NLP_translation_and_QnA_combined.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NLP translation and QnA combined.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "f7b6841ceeb94fd8baccc1378b136efb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0134a4de73e34b319609d697d721a9ca",
              "IPY_MODEL_fd186978d40b41c6a947e1c858f0d317",
              "IPY_MODEL_06c52196b2c348e8b3e5857af2fea720"
            ],
            "layout": "IPY_MODEL_79f6437888c44fe9947fd52be804ff05"
          }
        },
        "0134a4de73e34b319609d697d721a9ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1c1bfd6514fe43e3947635fc7d6b12b6",
            "placeholder": "​",
            "style": "IPY_MODEL_030055305316460a9ecb5e856845a2c2",
            "value": ""
          }
        },
        "fd186978d40b41c6a947e1c858f0d317": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f60aa94fe4024bd88759fac014422ae1",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1d0739ce70674985ab9b1c5904216e4f",
            "value": 0
          }
        },
        "06c52196b2c348e8b3e5857af2fea720": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2704e2268f7d466d85de300b472d4114",
            "placeholder": "​",
            "style": "IPY_MODEL_301ee3fa5bfa493887522e531ed9b7f6",
            "value": " 0/0 [00:00&lt;?, ?it/s]"
          }
        },
        "79f6437888c44fe9947fd52be804ff05": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1c1bfd6514fe43e3947635fc7d6b12b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "030055305316460a9ecb5e856845a2c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f60aa94fe4024bd88759fac014422ae1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "1d0739ce70674985ab9b1c5904216e4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2704e2268f7d466d85de300b472d4114": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "301ee3fa5bfa493887522e531ed9b7f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8723494408334a798d72ff90f2154f4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d07de9e100e14e16884a9777b30f5c24",
              "IPY_MODEL_710a49a963f74862861acf72e6da96bb",
              "IPY_MODEL_485ba94f2a5541c6bb35d399458e0399"
            ],
            "layout": "IPY_MODEL_a3660043b6714979839aa38b03883a35"
          }
        },
        "d07de9e100e14e16884a9777b30f5c24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_af4000d9da8e4486b1a52b89e19dfb23",
            "placeholder": "​",
            "style": "IPY_MODEL_27e8dd346cee4a2fbbba19f45c4ff330",
            "value": "Downloading builder script: "
          }
        },
        "710a49a963f74862861acf72e6da96bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_783b80461a5049d2aeb5fa2314c192be",
            "max": 1967,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fd51fbcc54bb467d8abc34692f45b8e5",
            "value": 1967
          }
        },
        "485ba94f2a5541c6bb35d399458e0399": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_985f7add00814ee680e3ece03341eceb",
            "placeholder": "​",
            "style": "IPY_MODEL_5de84f6775694337ad0ce5e74cc2b70f",
            "value": " 5.27k/? [00:00&lt;00:00, 15.3kB/s]"
          }
        },
        "a3660043b6714979839aa38b03883a35": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "af4000d9da8e4486b1a52b89e19dfb23": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27e8dd346cee4a2fbbba19f45c4ff330": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "783b80461a5049d2aeb5fa2314c192be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fd51fbcc54bb467d8abc34692f45b8e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "985f7add00814ee680e3ece03341eceb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5de84f6775694337ad0ce5e74cc2b70f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "acf3d909ed954ffba4c00c26209263c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c134ece7fc944ac498751a18a9a0d6e4",
              "IPY_MODEL_ab39d634862443aabff7ac5d367e6134",
              "IPY_MODEL_13f7ad747ec8466ba06f0ffa996a7ab7"
            ],
            "layout": "IPY_MODEL_e538b1347e354591ae01742447dc6e45"
          }
        },
        "c134ece7fc944ac498751a18a9a0d6e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c8a6165afd5445f19061dadcf7c91f9a",
            "placeholder": "​",
            "style": "IPY_MODEL_3f2ad9382d1442c1b110ccaefda4b6d5",
            "value": "Downloading metadata: "
          }
        },
        "ab39d634862443aabff7ac5d367e6134": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c73247bc954d4d469caa8eb7f2fb74ee",
            "max": 1021,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_685a04e9121848f0822c3a739151f0da",
            "value": 1021
          }
        },
        "13f7ad747ec8466ba06f0ffa996a7ab7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_634744632f2b47b8bee68db9f465e44a",
            "placeholder": "​",
            "style": "IPY_MODEL_0cc8153906c345c3b5bf8f1fc6f2e439",
            "value": " 2.36k/? [00:00&lt;00:00, 15.2kB/s]"
          }
        },
        "e538b1347e354591ae01742447dc6e45": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c8a6165afd5445f19061dadcf7c91f9a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3f2ad9382d1442c1b110ccaefda4b6d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c73247bc954d4d469caa8eb7f2fb74ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "685a04e9121848f0822c3a739151f0da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "634744632f2b47b8bee68db9f465e44a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0cc8153906c345c3b5bf8f1fc6f2e439": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1f301253c0804e3fba6b3ae2062131a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2f71f36153054398b0b831db1a25d8ea",
              "IPY_MODEL_75d0b99134ac403aac226f8061b77d4b",
              "IPY_MODEL_b01735bb14384c578130958ff4df2c00"
            ],
            "layout": "IPY_MODEL_c45debb358384259906b2556b00d59f4"
          }
        },
        "2f71f36153054398b0b831db1a25d8ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9ab8a25774114082a05657e59991f578",
            "placeholder": "​",
            "style": "IPY_MODEL_043e809e93b4426ba2eb4d74a43c85b6",
            "value": "Downloading data files: 100%"
          }
        },
        "75d0b99134ac403aac226f8061b77d4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_769646443fc14183983fefe66bc46661",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_488681387dee4869aa6e2d091dc7c52e",
            "value": 2
          }
        },
        "b01735bb14384c578130958ff4df2c00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4b775955ebdf4778aca4a44600950a9f",
            "placeholder": "​",
            "style": "IPY_MODEL_7853d3f6b7074c2192ff3e27a6471e4a",
            "value": " 2/2 [00:02&lt;00:00,  1.30s/it]"
          }
        },
        "c45debb358384259906b2556b00d59f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9ab8a25774114082a05657e59991f578": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "043e809e93b4426ba2eb4d74a43c85b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "769646443fc14183983fefe66bc46661": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "488681387dee4869aa6e2d091dc7c52e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4b775955ebdf4778aca4a44600950a9f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7853d3f6b7074c2192ff3e27a6471e4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f08e6971062f4e9babe03153dadc23be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d80417d9a84f4c27a0a76d7f5b5a66e6",
              "IPY_MODEL_9aa5344ee63e418d82d93dd7b9516d64",
              "IPY_MODEL_24908e30f3d446328b57fe5d3dad3431"
            ],
            "layout": "IPY_MODEL_1577c6513b19434f8bbce1e83028ecdf"
          }
        },
        "d80417d9a84f4c27a0a76d7f5b5a66e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_16b0955876f84291a2405d4dce139577",
            "placeholder": "​",
            "style": "IPY_MODEL_a590421d0dff4a9abfaf6cd705a10b45",
            "value": "Downloading data: "
          }
        },
        "9aa5344ee63e418d82d93dd7b9516d64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8022b4686459476688313ef5f51ec943",
            "max": 8116577,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_93249a4edf454b1e93fc55b2d9129346",
            "value": 8116577
          }
        },
        "24908e30f3d446328b57fe5d3dad3431": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6011b6345ac54a698d2001eeb98d12b9",
            "placeholder": "​",
            "style": "IPY_MODEL_84b8fc78995f4864aa7963dd0d740be1",
            "value": " 30.3M/? [00:01&lt;00:00, 30.8MB/s]"
          }
        },
        "1577c6513b19434f8bbce1e83028ecdf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16b0955876f84291a2405d4dce139577": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a590421d0dff4a9abfaf6cd705a10b45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8022b4686459476688313ef5f51ec943": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "93249a4edf454b1e93fc55b2d9129346": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6011b6345ac54a698d2001eeb98d12b9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "84b8fc78995f4864aa7963dd0d740be1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f0fdb0975b42482291329f1aad5a1129": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_752ad94afdaf4e98bb0f1ee41678cd6c",
              "IPY_MODEL_94b083619c684e658a4011a6de630316",
              "IPY_MODEL_5315d7095bbd4fcf9e6ebac71268d3f9"
            ],
            "layout": "IPY_MODEL_beefdb9b33714867a26eba13f55d6e23"
          }
        },
        "752ad94afdaf4e98bb0f1ee41678cd6c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_941a2af86f1448cea7339c44035740d6",
            "placeholder": "​",
            "style": "IPY_MODEL_b4658e88d2614710b09ddd5150821e57",
            "value": "Downloading data: "
          }
        },
        "94b083619c684e658a4011a6de630316": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4d53c1e5007443c486871da334884560",
            "max": 1054280,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_aa5e94bbb5ce4a7ebc637c4f3231d776",
            "value": 1054280
          }
        },
        "5315d7095bbd4fcf9e6ebac71268d3f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1a14533fc0fd483ba13fd0134a12ec21",
            "placeholder": "​",
            "style": "IPY_MODEL_91d64d3838b84508b262ee2e62857be0",
            "value": " 4.85M/? [00:00&lt;00:00, 12.6MB/s]"
          }
        },
        "beefdb9b33714867a26eba13f55d6e23": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "941a2af86f1448cea7339c44035740d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b4658e88d2614710b09ddd5150821e57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4d53c1e5007443c486871da334884560": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "aa5e94bbb5ce4a7ebc637c4f3231d776": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1a14533fc0fd483ba13fd0134a12ec21": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "91d64d3838b84508b262ee2e62857be0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f67611dfa6f34876b8e4b15851928afa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c469fa017bff4f6ca2ea6fda50400d02",
              "IPY_MODEL_98e0fbf82f5d4348953358426dcefe64",
              "IPY_MODEL_4a1a80c0bb7d47b08938fc7b6d3dd0a5"
            ],
            "layout": "IPY_MODEL_8923b4a5230c4d2aaee30c68d1dc113c"
          }
        },
        "c469fa017bff4f6ca2ea6fda50400d02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_224f8c4f66d648d8b6c2d4a16f0b834b",
            "placeholder": "​",
            "style": "IPY_MODEL_676c0fadb9904465a0b925e8b0d9b081",
            "value": "Extracting data files: 100%"
          }
        },
        "98e0fbf82f5d4348953358426dcefe64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9c0d14d0b71f44fab9ce41c6b2580a77",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_102cb8fe14f94ec8809db7b7abc45c24",
            "value": 2
          }
        },
        "4a1a80c0bb7d47b08938fc7b6d3dd0a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5ec75909c6a741d280cb4ab7487760cd",
            "placeholder": "​",
            "style": "IPY_MODEL_3d7faf2c09534f4388fed41eed8895bd",
            "value": " 2/2 [00:00&lt;00:00, 36.16it/s]"
          }
        },
        "8923b4a5230c4d2aaee30c68d1dc113c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "224f8c4f66d648d8b6c2d4a16f0b834b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "676c0fadb9904465a0b925e8b0d9b081": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9c0d14d0b71f44fab9ce41c6b2580a77": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "102cb8fe14f94ec8809db7b7abc45c24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5ec75909c6a741d280cb4ab7487760cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3d7faf2c09534f4388fed41eed8895bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d99ddf11f711439eae629174f7b0b4b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e65c1f73b59a45dc956b8e7b9b5fbdce",
              "IPY_MODEL_ce3cc67448d94c56866ba00dc9bac3f2",
              "IPY_MODEL_18db65a65f484e2bba03f91bcb35e39f"
            ],
            "layout": "IPY_MODEL_471127f3c0584de49a867eddb853ed88"
          }
        },
        "e65c1f73b59a45dc956b8e7b9b5fbdce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9c51c340389c434ba1a6391e54a449b6",
            "placeholder": "​",
            "style": "IPY_MODEL_20ab8ebebb4a407593704be5e79e96af",
            "value": "Generating train split:  99%"
          }
        },
        "ce3cc67448d94c56866ba00dc9bac3f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7a0f002068e24b38b854f3f6edad1d31",
            "max": 87599,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_539081123f254a2b81f793d25085738e",
            "value": 87599
          }
        },
        "18db65a65f484e2bba03f91bcb35e39f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b3df970fbf4044cba1ab02826bfd1ea5",
            "placeholder": "​",
            "style": "IPY_MODEL_df27d73400bc48cea3dc2e48d5308e5c",
            "value": " 86915/87599 [00:10&lt;00:00, 8865.65 examples/s]"
          }
        },
        "471127f3c0584de49a867eddb853ed88": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c51c340389c434ba1a6391e54a449b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "20ab8ebebb4a407593704be5e79e96af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7a0f002068e24b38b854f3f6edad1d31": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "539081123f254a2b81f793d25085738e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b3df970fbf4044cba1ab02826bfd1ea5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df27d73400bc48cea3dc2e48d5308e5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2647eff592574110b6e8e69cb6beb4f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_66b0c70a0097473dadf283003596bdaa",
              "IPY_MODEL_089d841d042b47a39cca766d34b7f92f",
              "IPY_MODEL_2a45dcec2630453ea0b268767fec3937"
            ],
            "layout": "IPY_MODEL_3ae9427a940a48faa28f577c2a3f05e1"
          }
        },
        "66b0c70a0097473dadf283003596bdaa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e516988ac1224cf699d2969240047e19",
            "placeholder": "​",
            "style": "IPY_MODEL_ed82461f4a624924835a6920d4614b85",
            "value": "Generating validation split:  92%"
          }
        },
        "089d841d042b47a39cca766d34b7f92f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8300de5a25944b5481702d23a6311a66",
            "max": 10570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8b17c09f331a4e3aab3a6a2da0fa2477",
            "value": 10570
          }
        },
        "2a45dcec2630453ea0b268767fec3937": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_db3279dc471447c7953f41b5767f8643",
            "placeholder": "​",
            "style": "IPY_MODEL_a51abf85ba084f998689e452d59e93fc",
            "value": " 9736/10570 [00:01&lt;00:00, 9735.50 examples/s]"
          }
        },
        "3ae9427a940a48faa28f577c2a3f05e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e516988ac1224cf699d2969240047e19": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed82461f4a624924835a6920d4614b85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8300de5a25944b5481702d23a6311a66": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8b17c09f331a4e3aab3a6a2da0fa2477": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "db3279dc471447c7953f41b5767f8643": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a51abf85ba084f998689e452d59e93fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a48f0a87b3aa4b2f9858160cb0bfd934": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a02fc8ed229146ac9d4dca57f5860a40",
              "IPY_MODEL_d74f1e4cab044a0ba63f22c33c74880c",
              "IPY_MODEL_22631c7b947642d1b9a90e223384a6ba"
            ],
            "layout": "IPY_MODEL_00f58419961e46d0a93c72d28b3cfc80"
          }
        },
        "a02fc8ed229146ac9d4dca57f5860a40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a7b8ea05ea7342b982531ef217489ea7",
            "placeholder": "​",
            "style": "IPY_MODEL_fa883d88bf3148f88c7918672a58c863",
            "value": "100%"
          }
        },
        "d74f1e4cab044a0ba63f22c33c74880c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8301dd16e4d046989d55d20be5231347",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5ee1a8cd815143539193a544d8b9fb53",
            "value": 2
          }
        },
        "22631c7b947642d1b9a90e223384a6ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0a0da0cc945145d7810e10651502c1c9",
            "placeholder": "​",
            "style": "IPY_MODEL_dc166a97f1cc43a5803955a473ed6b4b",
            "value": " 2/2 [00:00&lt;00:00, 34.87it/s]"
          }
        },
        "00f58419961e46d0a93c72d28b3cfc80": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a7b8ea05ea7342b982531ef217489ea7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fa883d88bf3148f88c7918672a58c863": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8301dd16e4d046989d55d20be5231347": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5ee1a8cd815143539193a544d8b9fb53": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0a0da0cc945145d7810e10651502c1c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dc166a97f1cc43a5803955a473ed6b4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# 1. Question answering model"
      ],
      "metadata": {
        "id": "f_DNYvumZbLN"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 1.1 Import modules\n",
        "\n"
      ],
      "metadata": {
        "id": "lTCxKKKqLDeO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers\n",
        "!pip install sentencepiece\n",
        "!pip install transformers[sentencepiece]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KIIwAiuLt6qn",
        "outputId": "68d78215-105a-47a2-fefd-9acfbdba2f2a"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.21.1-py3-none-any.whl (4.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.7 MB 14.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 11.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.12.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.1)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.8.1-py3-none-any.whl (101 kB)\n",
            "\u001b[K     |████████████████████████████████| 101 kB 6.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 28.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.6.15)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Installing collected packages: pyyaml, tokenizers, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.8.1 pyyaml-6.0 tokenizers-0.12.1 transformers-4.21.1\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.97-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.3 MB 23.3 MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.97\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers[sentencepiece] in /usr/local/lib/python3.7/dist-packages (4.21.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (3.7.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (1.21.6)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (0.12.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (6.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (4.12.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (4.64.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (2022.6.2)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (0.8.1)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (21.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (2.23.0)\n",
            "Requirement already satisfied: protobuf<=3.20.1 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (3.17.3)\n",
            "Requirement already satisfied: sentencepiece!=0.1.92,>=0.1.91 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (0.1.97)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers[sentencepiece]) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers[sentencepiece]) (3.0.9)\n",
            "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.7/dist-packages (from protobuf<=3.20.1->transformers[sentencepiece]) (1.15.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers[sentencepiece]) (3.8.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers[sentencepiece]) (2022.6.15)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers[sentencepiece]) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers[sentencepiece]) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers[sentencepiece]) (1.24.3)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "0b3d4e18",
        "outputId": "5eea9c52-91a0-4efd-a19c-827bc3015062",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.21.1)\n",
            "Collecting datasets\n",
            "  Downloading datasets-2.4.0-py3-none-any.whl (365 kB)\n",
            "\u001b[K     |████████████████████████████████| 365 kB 26.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: huggingface_hub in /usr/local/lib/python3.7/dist-packages (0.8.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.12.0)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.12.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface_hub) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Collecting fsspec[http]>=2021.11.1\n",
            "  Downloading fsspec-2022.7.1-py3-none-any.whl (141 kB)\n",
            "\u001b[K     |████████████████████████████████| 141 kB 59.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (from datasets) (3.8.1)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0.1)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[K     |████████████████████████████████| 212 kB 55.1 MB/s \n",
            "\u001b[?25hCollecting multiprocess\n",
            "  Downloading multiprocess-0.70.13-py37-none-any.whl (115 kB)\n",
            "\u001b[K     |████████████████████████████████| 115 kB 66.2 MB/s \n",
            "\u001b[?25hCollecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: dill<0.3.6 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.5.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.6.15)\n",
            "Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1\n",
            "  Downloading urllib3-1.25.11-py2.py3-none-any.whl (127 kB)\n",
            "\u001b[K     |████████████████████████████████| 127 kB 69.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.2.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (6.0.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (22.1.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.1.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.8.1)\n",
            "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (0.13.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.1)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2022.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: urllib3, fsspec, xxhash, responses, multiprocess, datasets\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "Successfully installed datasets-2.4.0 fsspec-2022.7.1 multiprocess-0.70.13 responses-0.18.0 urllib3-1.25.11 xxhash-3.0.0\n"
          ]
        }
      ],
      "source": [
        "! pip install transformers datasets huggingface_hub"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "ed7b6d2e",
        "outputId": "695275a8-9314-4e5b-c31c-d20cb27fd2fc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1minfo:\u001b[0m downloading installer\n",
            "sh: 147: cannot open /dev/tty: No such device or address\n",
            "Cloning into 'tokenizers'...\n",
            "remote: Enumerating objects: 15558, done.\u001b[K\n",
            "remote: Counting objects: 100% (2315/2315), done.\u001b[K\n",
            "remote: Compressing objects: 100% (1069/1069), done.\u001b[K\n",
            "remote: Total 15558 (delta 1518), reused 1838 (delta 1207), pack-reused 13243\u001b[K\n",
            "Receiving objects: 100% (15558/15558), 7.76 MiB | 14.12 MiB/s, done.\n",
            "Resolving deltas: 100% (10151/10151), done.\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting setuptools_rust\n",
            "  Downloading setuptools_rust-1.5.0-py3-none-any.whl (23 kB)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from setuptools_rust) (4.1.1)\n",
            "Collecting semantic-version<3,>=2.8.2\n",
            "  Downloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n",
            "Collecting setuptools>=62.4\n",
            "  Downloading setuptools-64.0.3-py3-none-any.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 9.7 MB/s \n",
            "\u001b[?25hInstalling collected packages: setuptools, semantic-version, setuptools-rust\n",
            "  Attempting uninstall: setuptools\n",
            "    Found existing installation: setuptools 57.4.0\n",
            "    Uninstalling setuptools-57.4.0:\n",
            "      Successfully uninstalled setuptools-57.4.0\n",
            "Successfully installed semantic-version-2.10.0 setuptools-64.0.3 setuptools-rust-1.5.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "pkg_resources"
                ]
              }
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "python3: can't open file 'setup.py': [Errno 2] No such file or directory\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting git+https://github.com/huggingface/transformers\n",
            "  Cloning https://github.com/huggingface/transformers to /tmp/pip-req-build-xj46oh0v\n",
            "  Running command git clone -q https://github.com/huggingface/transformers /tmp/pip-req-build-xj46oh0v\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers==4.22.0.dev0) (6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.22.0.dev0) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.22.0.dev0) (1.21.6)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from transformers==4.22.0.dev0) (0.8.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.22.0.dev0) (3.7.1)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers==4.22.0.dev0) (0.12.1)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers==4.22.0.dev0) (4.12.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers==4.22.0.dev0) (21.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==4.22.0.dev0) (4.64.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.22.0.dev0) (2022.6.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.8.1->transformers==4.22.0.dev0) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers==4.22.0.dev0) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers==4.22.0.dev0) (3.8.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.22.0.dev0) (2022.6.15)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.22.0.dev0) (1.25.11)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.22.0.dev0) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.22.0.dev0) (2.10)\n",
            "Building wheels for collected packages: transformers\n",
            "  Building wheel for transformers (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-4.22.0.dev0-py3-none-any.whl size=4728457 sha256=af2896c3e1fe2abe9b30c0508c4d40a7333e9000c37cb0576d8e557c21c5b221\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-8wziwz3u/wheels/35/2e/a7/d819e3310040329f0f47e57c9e3e7a7338aa5e74c49acfe522\n",
            "Successfully built transformers\n",
            "Installing collected packages: transformers\n",
            "  Attempting uninstall: transformers\n",
            "    Found existing installation: transformers 4.21.1\n",
            "    Uninstalling transformers-4.21.1:\n",
            "      Successfully uninstalled transformers-4.21.1\n",
            "Successfully installed transformers-4.22.0.dev0\n"
          ]
        }
      ],
      "source": [
        "!curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh\n",
        "!git clone https://github.com/huggingface/tokenizers\n",
        "!pip install setuptools_rust\n",
        "!python setup.py install\n",
        "!pip install git+https://github.com/huggingface/transformers\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "1b574801",
        "outputId": "bcf7590e-a6b2-4e65-af5f-ac5cf03798e7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 103,
          "referenced_widgets": [
            "f7b6841ceeb94fd8baccc1378b136efb",
            "0134a4de73e34b319609d697d721a9ca",
            "fd186978d40b41c6a947e1c858f0d317",
            "06c52196b2c348e8b3e5857af2fea720",
            "79f6437888c44fe9947fd52be804ff05",
            "1c1bfd6514fe43e3947635fc7d6b12b6",
            "030055305316460a9ecb5e856845a2c2",
            "f60aa94fe4024bd88759fac014422ae1",
            "1d0739ce70674985ab9b1c5904216e4f",
            "2704e2268f7d466d85de300b472d4114",
            "301ee3fa5bfa493887522e531ed9b7f6"
          ]
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The cache for model files in Transformers v4.22.0 has been updated. Migrating your old cache. This is a one-time only operation. You can interrupt this and resume the migration later on by calling `transformers.utils.move_cache()`.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Moving 0 files to the new cache system\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "0it [00:00, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f7b6841ceeb94fd8baccc1378b136efb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "4.22.0.dev0\n"
          ]
        }
      ],
      "source": [
        "import transformers\n",
        "\n",
        "print(transformers.__version__)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 1.2 Load Dataset\n"
      ],
      "metadata": {
        "id": "_wJe1vxTLn6X"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "571c1629"
      },
      "outputs": [],
      "source": [
        "squad_v2 = False\n",
        "model_checkpoint = \"distilbert-base-uncased\"\n",
        "batch_size = 16"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "d65aa004"
      },
      "outputs": [],
      "source": [
        "from datasets import load_dataset, load_metric"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "datasets = load_dataset(\"squad_v2\" if squad_v2 else \"squad\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297,
          "referenced_widgets": [
            "8723494408334a798d72ff90f2154f4b",
            "d07de9e100e14e16884a9777b30f5c24",
            "710a49a963f74862861acf72e6da96bb",
            "485ba94f2a5541c6bb35d399458e0399",
            "a3660043b6714979839aa38b03883a35",
            "af4000d9da8e4486b1a52b89e19dfb23",
            "27e8dd346cee4a2fbbba19f45c4ff330",
            "783b80461a5049d2aeb5fa2314c192be",
            "fd51fbcc54bb467d8abc34692f45b8e5",
            "985f7add00814ee680e3ece03341eceb",
            "5de84f6775694337ad0ce5e74cc2b70f",
            "acf3d909ed954ffba4c00c26209263c0",
            "c134ece7fc944ac498751a18a9a0d6e4",
            "ab39d634862443aabff7ac5d367e6134",
            "13f7ad747ec8466ba06f0ffa996a7ab7",
            "e538b1347e354591ae01742447dc6e45",
            "c8a6165afd5445f19061dadcf7c91f9a",
            "3f2ad9382d1442c1b110ccaefda4b6d5",
            "c73247bc954d4d469caa8eb7f2fb74ee",
            "685a04e9121848f0822c3a739151f0da",
            "634744632f2b47b8bee68db9f465e44a",
            "0cc8153906c345c3b5bf8f1fc6f2e439",
            "1f301253c0804e3fba6b3ae2062131a6",
            "2f71f36153054398b0b831db1a25d8ea",
            "75d0b99134ac403aac226f8061b77d4b",
            "b01735bb14384c578130958ff4df2c00",
            "c45debb358384259906b2556b00d59f4",
            "9ab8a25774114082a05657e59991f578",
            "043e809e93b4426ba2eb4d74a43c85b6",
            "769646443fc14183983fefe66bc46661",
            "488681387dee4869aa6e2d091dc7c52e",
            "4b775955ebdf4778aca4a44600950a9f",
            "7853d3f6b7074c2192ff3e27a6471e4a",
            "f08e6971062f4e9babe03153dadc23be",
            "d80417d9a84f4c27a0a76d7f5b5a66e6",
            "9aa5344ee63e418d82d93dd7b9516d64",
            "24908e30f3d446328b57fe5d3dad3431",
            "1577c6513b19434f8bbce1e83028ecdf",
            "16b0955876f84291a2405d4dce139577",
            "a590421d0dff4a9abfaf6cd705a10b45",
            "8022b4686459476688313ef5f51ec943",
            "93249a4edf454b1e93fc55b2d9129346",
            "6011b6345ac54a698d2001eeb98d12b9",
            "84b8fc78995f4864aa7963dd0d740be1",
            "f0fdb0975b42482291329f1aad5a1129",
            "752ad94afdaf4e98bb0f1ee41678cd6c",
            "94b083619c684e658a4011a6de630316",
            "5315d7095bbd4fcf9e6ebac71268d3f9",
            "beefdb9b33714867a26eba13f55d6e23",
            "941a2af86f1448cea7339c44035740d6",
            "b4658e88d2614710b09ddd5150821e57",
            "4d53c1e5007443c486871da334884560",
            "aa5e94bbb5ce4a7ebc637c4f3231d776",
            "1a14533fc0fd483ba13fd0134a12ec21",
            "91d64d3838b84508b262ee2e62857be0",
            "f67611dfa6f34876b8e4b15851928afa",
            "c469fa017bff4f6ca2ea6fda50400d02",
            "98e0fbf82f5d4348953358426dcefe64",
            "4a1a80c0bb7d47b08938fc7b6d3dd0a5",
            "8923b4a5230c4d2aaee30c68d1dc113c",
            "224f8c4f66d648d8b6c2d4a16f0b834b",
            "676c0fadb9904465a0b925e8b0d9b081",
            "9c0d14d0b71f44fab9ce41c6b2580a77",
            "102cb8fe14f94ec8809db7b7abc45c24",
            "5ec75909c6a741d280cb4ab7487760cd",
            "3d7faf2c09534f4388fed41eed8895bd",
            "d99ddf11f711439eae629174f7b0b4b1",
            "e65c1f73b59a45dc956b8e7b9b5fbdce",
            "ce3cc67448d94c56866ba00dc9bac3f2",
            "18db65a65f484e2bba03f91bcb35e39f",
            "471127f3c0584de49a867eddb853ed88",
            "9c51c340389c434ba1a6391e54a449b6",
            "20ab8ebebb4a407593704be5e79e96af",
            "7a0f002068e24b38b854f3f6edad1d31",
            "539081123f254a2b81f793d25085738e",
            "b3df970fbf4044cba1ab02826bfd1ea5",
            "df27d73400bc48cea3dc2e48d5308e5c",
            "2647eff592574110b6e8e69cb6beb4f2",
            "66b0c70a0097473dadf283003596bdaa",
            "089d841d042b47a39cca766d34b7f92f",
            "2a45dcec2630453ea0b268767fec3937",
            "3ae9427a940a48faa28f577c2a3f05e1",
            "e516988ac1224cf699d2969240047e19",
            "ed82461f4a624924835a6920d4614b85",
            "8300de5a25944b5481702d23a6311a66",
            "8b17c09f331a4e3aab3a6a2da0fa2477",
            "db3279dc471447c7953f41b5767f8643",
            "a51abf85ba084f998689e452d59e93fc",
            "a48f0a87b3aa4b2f9858160cb0bfd934",
            "a02fc8ed229146ac9d4dca57f5860a40",
            "d74f1e4cab044a0ba63f22c33c74880c",
            "22631c7b947642d1b9a90e223384a6ba",
            "00f58419961e46d0a93c72d28b3cfc80",
            "a7b8ea05ea7342b982531ef217489ea7",
            "fa883d88bf3148f88c7918672a58c863",
            "8301dd16e4d046989d55d20be5231347",
            "5ee1a8cd815143539193a544d8b9fb53",
            "0a0da0cc945145d7810e10651502c1c9",
            "dc166a97f1cc43a5803955a473ed6b4b"
          ]
        },
        "id": "AnCC3to6GNko",
        "outputId": "db2965d7-1856-42fd-9d6e-4c0dee5ebc00"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/1.97k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8723494408334a798d72ff90f2154f4b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading metadata:   0%|          | 0.00/1.02k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "acf3d909ed954ffba4c00c26209263c0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset squad/plain_text (download: 33.51 MiB, generated: 85.63 MiB, post-processed: Unknown size, total: 119.14 MiB) to /root/.cache/huggingface/datasets/squad/plain_text/1.0.0/d6ec3ceb99ca480ce37cdd35555d6cb2511d223b9150cce08a837ef62ffea453...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data files:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1f301253c0804e3fba6b3ae2062131a6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/8.12M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f08e6971062f4e9babe03153dadc23be"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/1.05M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f0fdb0975b42482291329f1aad5a1129"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Extracting data files:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f67611dfa6f34876b8e4b15851928afa"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/87599 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d99ddf11f711439eae629174f7b0b4b1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating validation split:   0%|          | 0/10570 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2647eff592574110b6e8e69cb6beb4f2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset squad downloaded and prepared to /root/.cache/huggingface/datasets/squad/plain_text/1.0.0/d6ec3ceb99ca480ce37cdd35555d6cb2511d223b9150cce08a837ef62ffea453. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a48f0a87b3aa4b2f9858160cb0bfd934"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6ad9ee01",
        "outputId": "c40a5024-dfac-4af2-c75b-7aec4070815e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['id', 'title', 'context', 'question', 'answers'],\n",
              "        num_rows: 87599\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['id', 'title', 'context', 'question', 'answers'],\n",
              "        num_rows: 10570\n",
              "    })\n",
              "})"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "datasets"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "datasets = load_dataset(\"squad_v2\" if squad_v2 else \"squad\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297,
          "referenced_widgets": [
            "d6aa67e1b137407ca18839687c14aaba",
            "09334c3a2628407784a0181879d0dce8",
            "21163ca65d0e48f797a96d1c30bb9262",
            "908610f482d6449c97bfc67967d6173d",
            "13aefe733ff943ce8cc53f8485be13d2",
            "80511261d99e4ada9bd20b12c462fa5f",
            "2f34d141a6674df0a392116f8909d572",
            "81233efd044e44c99323e1978aa9e390",
            "2989646f8b4a4b70b481c34b4de0a0c3",
            "3cecc77e8f5a4622b4d0ae19eec3f6de",
            "8f0518f21afd438eafea687f9809eca4",
            "cd36c37d4cea4ccd8dab468c105a8330",
            "24e4cd35876045e2a76569a89847cada",
            "caee686cfcf94ef89bc66530ad26c410",
            "bc0f2f55ad6b4d249b27d4fa0ed2f7b6",
            "7707125fd55f4a2bbffb6e2a095b3e0e",
            "50e8e836325b4aaeb075366969cff568",
            "8ce2a4d2222c46b99886b6bae458dd99",
            "f4d751f451064f48866b2427af5538cd",
            "8a733440856c4acc8caaae2f58583d15",
            "d2628569fdf6498eab4cebb1c111b934",
            "3ddf1f200ec3431d8c2e2cb24366a2d7",
            "76eabf8ff39a46618bd557eec1367000",
            "f8bf8fd04db748b2bec2f8cb49891d76",
            "09df5a2c4d4b4f998da1f2936972fd37",
            "c91aea0e321f4e06bef86634703f22c8",
            "98791fa68db2465fb8ba95dabf2054e3",
            "a0c2b4f50cff4b58ad0f3f9b317ffe21",
            "e711b8ca359a49acab1b37f71ec1cf71",
            "9b9c0d1b24a24979b57b915e11595801",
            "cc7dea63fc3e44b9a53ab27deda94fde",
            "0359293071fd46c0a36d89db3c1f5926",
            "4fed5275fefd4aeb94e3d5ad38273873",
            "5a058f361a6640e6924bf8bde9009b69",
            "0efa6492351945209e2342cc3833d547",
            "d9f72bb70262414091a5b5ecc9dccd6c",
            "a352c016968845829eee167ee9c8ef09",
            "9803bf54de77463281d310472661afb4",
            "7f8c237f23914a35a4ddc622f8d29628",
            "3d8a609c62a2468681ab9b6696066d42",
            "5a8ce91c67eb4fb4829f50277984676e",
            "b82e1a784055459687a5bc33e5980af1",
            "04ff79828c7b47799e3734ab3a1704c9",
            "19c3d177d22b4342b6a887893fe7c63b",
            "79f7b3bf120a474c882040e43a49eb54",
            "f43d9ed789724e3696cd92bd6fa2aac5",
            "4941ec0145604ccd85e4e3a39f7ef143",
            "86427c9d58eb4d05a6fd9fbd0a388f3b",
            "b83ebced8d9046a09c8046f42f47ccba",
            "fefe39b493ef40e0b536aa5a4fc21e06",
            "eef263d0da19489cbb369d161c3e5eec",
            "57b8b73b5c2948e991ff2905f57894b6",
            "2ff4745e5c1e4472b3f2478f56e9b560",
            "96060b402cb24584b04f00dde12c2e1a",
            "9d87280d676a4832a65cb4abea5a9249",
            "b85d784428a04b81974fdf135812c7c4",
            "9109b0b399f84cdda64b8743cad2f4a9",
            "c7fd99fc81c049d8974f8ffd5e0557ae",
            "05ea6e49db224ef38cb9a44fdc6725cd",
            "085ef4cc9856436e82ea488aab721414",
            "ab2e5cda2ade4097965f7e14ed43d50a",
            "b5e43576042744ad986b06811bd9ad83",
            "d21b6594f83041a69a40cc40909235e1",
            "4379d2b3f3fe4c70b8145046791c116c",
            "23a381876d024c37a89e0fdbcf7f3b30",
            "29cd2935711145e1bf5f26a11bb6816f",
            "210290302ecf4fcebc0c66c53fc633a8",
            "418fd0ce655d4ac4a45885ccb581c8ad",
            "379c8d43f2f04240a9831da305502864",
            "bca9e74daef546918c58030cff8c2dd2",
            "fa2136ff4c204a4bafcad0dd87350d1a",
            "c02ff191787d42e8889cd2a35079bc4f",
            "11d6ec5c464141709289e61eee7c6c85",
            "15bffb2cd7fd41d7911e53932a2fcd8d",
            "efbfabcbeb5943ebbf8e41de64c118ae",
            "74a417c53b69435dae7b941b47155bb6",
            "56ffd6b7bf404232afd0b5a7781ff0ac",
            "f27a5646cb3549e091116edfc30a00dd",
            "da2afb3ed2ff4169940ec6f2d2cf0c69",
            "7a13700010d24a599a6b018fd4c8f7f9",
            "681bbabc6f8b4c588fa7f676ff3492b8",
            "768caaff18044ef8b161d9a416fa098b",
            "700a33e69b8b4960b19c3fb21578bc83",
            "2b46f805f5d84f3a94f8d67064d8fa5c",
            "383693420cf546c895875fac98ac0e28",
            "fb24b0addb0a4dc2a540fe5c32583507",
            "c0286ac935f04d1b88d66f161ff0f3ff",
            "a7468ebe2aaf473d80790293c19a5747",
            "22532aeb55494441aeff0ac66ed13285",
            "0543a3839e944c1194dfa1375c4f6565",
            "fe7e7efde01b42059fead4c76654693c",
            "a73e143706894ebd902cf0852cd5502d",
            "13b57016e4b5449b8c6b6088384e7316",
            "3f847af4468349ec9329fd00f8a0e04f",
            "755c165a40ba4c3db12aca25a50dd8ca",
            "d734e1af57b94eb9b2924eb69a06f62a",
            "cdd6ad62bcb249c99e087c0a66f25565",
            "73611487ffba410ea8f1518774a88a63",
            "684a8c0fdaf34e50a8aec972328b8895"
          ]
        },
        "id": "P_-YYnzYNQ8f",
        "outputId": "4b6a89d9-673b-49ee-a495-f979bbcef5be"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/1.97k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d6aa67e1b137407ca18839687c14aaba"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading metadata:   0%|          | 0.00/1.02k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cd36c37d4cea4ccd8dab468c105a8330"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset squad/plain_text (download: 33.51 MiB, generated: 85.63 MiB, post-processed: Unknown size, total: 119.14 MiB) to /root/.cache/huggingface/datasets/squad/plain_text/1.0.0/d6ec3ceb99ca480ce37cdd35555d6cb2511d223b9150cce08a837ef62ffea453...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data files:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "76eabf8ff39a46618bd557eec1367000"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/8.12M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5a058f361a6640e6924bf8bde9009b69"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/1.05M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "79f7b3bf120a474c882040e43a49eb54"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Extracting data files:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b85d784428a04b81974fdf135812c7c4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/87599 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "210290302ecf4fcebc0c66c53fc633a8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating validation split:   0%|          | 0/10570 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f27a5646cb3549e091116edfc30a00dd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset squad downloaded and prepared to /root/.cache/huggingface/datasets/squad/plain_text/1.0.0/d6ec3ceb99ca480ce37cdd35555d6cb2511d223b9150cce08a837ef62ffea453. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "22532aeb55494441aeff0ac66ed13285"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "19a20208",
        "outputId": "f5f178a8-fc28-4a67-cb76-a22691128a26"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'id': '5733be284776f41900661182',\n",
              " 'title': 'University_of_Notre_Dame',\n",
              " 'context': 'Architecturally, the school has a Catholic character. Atop the Main Building\\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.',\n",
              " 'question': 'To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?',\n",
              " 'answers': {'text': ['Saint Bernadette Soubirous'], 'answer_start': [515]}}"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "datasets[\"train\"][0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e68f5e09"
      },
      "outputs": [],
      "source": [
        "from datasets import ClassLabel, Sequence\n",
        "import random\n",
        "import pandas as pd\n",
        "from IPython.display import display, HTML\n",
        "\n",
        "\n",
        "def show_random_elements(dataset, num_examples=10):\n",
        "    assert num_examples <= len(\n",
        "        dataset\n",
        "    ), \"Can't pick more elements than there are in the dataset.\"\n",
        "    picks = []\n",
        "    for _ in range(num_examples):\n",
        "        pick = random.randint(0, len(dataset) - 1)\n",
        "        while pick in picks:\n",
        "            pick = random.randint(0, len(dataset) - 1)\n",
        "        picks.append(pick)\n",
        "\n",
        "    df = pd.DataFrame(dataset[picks])\n",
        "    for column, typ in dataset.features.items():\n",
        "        if isinstance(typ, ClassLabel):\n",
        "            df[column] = df[column].transform(lambda i: typ.names[i])\n",
        "        elif isinstance(typ, Sequence) and isinstance(typ.feature, ClassLabel):\n",
        "            df[column] = df[column].transform(\n",
        "                lambda x: [typ.feature.names[i] for i in x]\n",
        "            )\n",
        "    display(HTML(df.to_html()))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4f02e0be",
        "outputId": "f63b2a96-3e7d-49f8-bc21-c0d1a7d41aee"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>title</th>\n",
              "      <th>context</th>\n",
              "      <th>question</th>\n",
              "      <th>answers</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>56d385c459d6e41400146629</td>\n",
              "      <td>American_Idol</td>\n",
              "      <td>American Idol premiered in June 2002 and became the surprise summer hit show of 2002. The first show drew 9.9 million viewers, giving Fox the best viewing figure for the 8.30 pm spot in over a year. The audience steadily grew, and by finale night, the audience had averaged 23 million, with more than 40 million watching some part of that show. That episode was placed third amongst all age groups, but more importantly it led in the 18–49 demographic, the age group most valued by advertisers.</td>\n",
              "      <td>What year did American Idol being to air on television?</td>\n",
              "      <td>{'text': ['2002'], 'answer_start': [32]}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>56cf4e5faab44d1400b88f92</td>\n",
              "      <td>Frédéric_Chopin</td>\n",
              "      <td>Chopin's life and his relations with George Sand have been fictionalized in numerous films. The 1945 biographical film A Song to Remember earned Cornel Wilde an Academy Award nomination as Best Actor for his portrayal of the composer. Other film treatments have included: La valse de l'adieu (France, 1928) by Henry Roussel, with Pierre Blanchar as Chopin; Impromptu (1991), starring Hugh Grant as Chopin; La note bleue (1991); and Chopin: Desire for Love (2002).</td>\n",
              "      <td>What year was  La valse de l'adieu released?</td>\n",
              "      <td>{'text': ['1928'], 'answer_start': [301]}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>573055ee8ab72b1400f9c47e</td>\n",
              "      <td>Windows_8</td>\n",
              "      <td>In May 2014, the Government of China banned the internal purchase of Windows 8-based products under government contracts requiring \"energy-efficient\" devices. The Xinhua News Agency claimed that Windows 8 was being banned in protest of Microsoft's support lifecycle policy and the end of support for Windows XP (which, as of January 2014, had a market share of 49% in China), as the government \"obviously cannot ignore the risks of running OS [sic] without guaranteed technical support.\" However, Ni Guangnan of the Chinese Academy of Sciences had also previously warned that Windows 8 could allegedly expose users to surveillance by the United States government due to its heavy use of internet-based services.</td>\n",
              "      <td>When did the Chinese government ban the purchase of Windows 8 products?</td>\n",
              "      <td>{'text': ['May 2014'], 'answer_start': [3]}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>5727b9224b864d1900163b7b</td>\n",
              "      <td>Tibet</td>\n",
              "      <td>Tibet (i/tᵻˈbɛt/; Wylie: Bod, pronounced [pʰø̀ʔ]; Chinese: 西藏; pinyin: Xīzàng) is a region on the Tibetan Plateau in Asia. It is the traditional homeland of the Tibetan people as well as some other ethnic groups such as Monpa, Qiang and Lhoba peoples and is now also inhabited by considerable numbers of Han Chinese and Hui people. Tibet is the highest region on Earth, with an average elevation of 4,900 metres (16,000 ft). The highest elevation in Tibet is Mount Everest, earth's highest mountain rising 8,848 m (29,029 ft) above sea level.</td>\n",
              "      <td>Which continent contains Tibet?</td>\n",
              "      <td>{'text': ['Asia'], 'answer_start': [117]}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5725fa9c38643c19005acf39</td>\n",
              "      <td>Hellenistic_period</td>\n",
              "      <td>As a result of the confusion in Greece at the end of the Second Macedonian War, the Seleucid Empire also became entangled with the Romans. The Seleucid Antiochus III had allied with Philip V of Macedon in 203 BC, agreeing that they should jointly conquer the lands of the boy-king of Egypt, Ptolemy V. After defeating Ptolemy in the Fifth Syrian War, Antiochus concentrated on occupying the Ptolemaic possessions in Asia Minor. However, this brought Antiochus into conflict with Rhodes and Pergamum, two important Roman allies, and began a 'cold war' between Rome and Antiochus (not helped by the presence of Hannibal at the Seleucid court). Meanwhile, in mainland Greece, the Aetolian League, which had sided with Rome against Macedon, now grew to resent the Roman presence in Greece. This presented Antiochus III with a pretext to invade Greece and 'liberate' it from Roman influence, thus starting the Roman-Syrian War (192–188 BC). In 191 BC, the Romans under Manius Acilius Glabrio routed him at Thermopylae and obliged him to withdraw to Asia. During the course of this war Roman troops moved into Asia for the first time, where they defeated Antiochus again at the Battle of Magnesia (190 BC). A crippling treaty was imposed on Antiochus, with Seleucid possessions in Asia Minor removed and given to Rhodes and Pergamum, the size of the Seleucid navy reduced, and a massive war indemnity invoked.</td>\n",
              "      <td>What years did the Roman-Syrian war take place?</td>\n",
              "      <td>{'text': ['192–188 BC'], 'answer_start': [923]}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>56d39ed559d6e41400146825</td>\n",
              "      <td>Frédéric_Chopin</td>\n",
              "      <td>The two mature piano sonatas (No. 2, Op. 35, written in 1839 and No. 3, Op. 58, written in 1844) are in four movements. In Op. 35, Chopin was able to combine within a formal large musical structure many elements of his virtuosic piano technique—\"a kind of dialogue between the public pianism of the brilliant style and the German sonata principle\". The last movement, a brief (75-bar) perpetuum mobile in which the hands play in unmodified octave unison throughout, was found shocking and unmusical by contemporaries, including Schumann. The Op. 58 sonata is closer to the German tradition, including many passages of complex counterpoint, \"worthy of Brahms\" according to the music historians Kornel Michałowski and Jim Samson.</td>\n",
              "      <td>How many movements are in No. 2, Op. 35 and No. 3, Op 58?</td>\n",
              "      <td>{'text': ['four'], 'answer_start': [104]}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>572febaaa23a5019007fcb3f</td>\n",
              "      <td>Bacteria</td>\n",
              "      <td>Bacteria were first observed by the Dutch microscopist Antonie van Leeuwenhoek in 1676, using a single-lens microscope of his own design. He then published his observations in a series of letters to the Royal Society of London. Bacteria were Leeuwenhoek's most remarkable microscopic discovery. They were just at the limit of what his simple lenses could make out and, in one of the most striking hiatuses in the history of science, no one else would see them again for over a century. Only then were his by-then-largely-forgotten observations of bacteria — as opposed to his famous \"animalcules\" (spermatozoa) — taken seriously.</td>\n",
              "      <td>When was the first time bacteria were observed?</td>\n",
              "      <td>{'text': ['1676'], 'answer_start': [82]}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>57273da05951b619008f8733</td>\n",
              "      <td>Affirmative_action_in_the_United_States</td>\n",
              "      <td>Following the end of World War II the educational gap between White and Black Americans was widened by Dwight D. Eisenhower's GI Bill. This piece of legislation paved the way for white GIs to attend college. Despite their veteran status returning black servicemen were not afforded loans at the same rate as whites. Furthermore, at the time of its introduction, segregation was still the law of the land barring blacks from the best institutions. Overall, \"Nearly 8 million servicemen and servicewomen were educated under the provisions of the GI Bill after World War II. But for blacks, higher educational opportunities were so few that the promise of the GI Bill went largely unfulfilled.\"</td>\n",
              "      <td>What major event preceded the growth of the divide in education between White and Black Americans?</td>\n",
              "      <td>{'text': ['end of World War II'], 'answer_start': [14]}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>56de58b94396321400ee283b</td>\n",
              "      <td>Southern_Europe</td>\n",
              "      <td>For its official works and publications, the United Nations Organization groups countries under a classification of regions. The assignment of countries or areas to specific groupings is for statistical convenience and does not imply any assumption regarding political or other affiliation of countries or territories by the United Nations. Southern Europe, as grouped for statistical convenience by the United Nations (the sub-regions according to the UN), includes following countries and territories:</td>\n",
              "      <td>What term is used by the UNO to divide groups of nations?</td>\n",
              "      <td>{'text': ['regions'], 'answer_start': [116]}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>571d3d015efbb31900334edc</td>\n",
              "      <td>Pharmaceutical_industry</td>\n",
              "      <td>Others have argued that excessive regulation suppresses therapeutic innovation, and that the current cost of regulator-required clinical trials prevents the full exploitation of new genetic and biological knowledge for the treatment of human disease. A 2012 report by the President's Council of Advisors on Science and Technology made several key recommendations to reduce regulatory burdens to new drug development, including 1) expanding the FDA's use of accelerated approval processes, 2) creating an expedited approval pathway for drugs intended for use in narrowly defined populations, and 3) undertaking pilot projects designed to evaluate the feasibility of a new, adaptive drug approval process.</td>\n",
              "      <td>What was argued to be holding back new knowledge for treating diseases?</td>\n",
              "      <td>{'text': ['current cost of regulator-required clinical trials'], 'answer_start': [93]}</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "ename": "AttributeError",
          "evalue": "'NoneType' object has no attribute 'head'",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "Input \u001b[0;32mIn [18]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mshow_random_elements\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdatasets\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtrain\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhead\u001b[49m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'head'"
          ]
        }
      ],
      "source": [
        "show_random_elements(datasets[\"train\"]).head\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 1.3 Tokenize input"
      ],
      "metadata": {
        "id": "56t9wxvHL6ED"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "00e02b50",
        "outputId": "99293b61-0e99-4076-e87e-4cd471b44a46"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'input_ids': [101, 2054, 2003, 2115, 2171, 1029, 102, 2026, 2171, 2003, 25353, 22144, 2378, 1012, 102], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "tokenizer(\"What is your name?\", \"My name is Sylvain.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "701236ff"
      },
      "outputs": [],
      "source": [
        "for i, example in enumerate(datasets[\"train\"]):\n",
        "    if len(tokenizer(example[\"question\"], example[\"context\"])[\"input_ids\"]) > 384:\n",
        "        break\n",
        "example = datasets[\"train\"][i]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "71b4a05f",
        "outputId": "2cf2741e-190e-4373-94b3-e06bd6c00ed8"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "396"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(tokenizer(example[\"question\"], example[\"context\"])[\"input_ids\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "28583cd9"
      },
      "outputs": [],
      "source": [
        "max_length = 384  # The maximum length of a feature (question and context)\n",
        "doc_stride = 128  # The allowed overlap between two part of the context when splitting is performed."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7714ddb0",
        "outputId": "d02965f2-be3e-4a73-d4ae-da1772a47302"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "384"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(\n",
        "    tokenizer(\n",
        "        example[\"question\"],\n",
        "        example[\"context\"],\n",
        "        max_length=max_length,\n",
        "        truncation=\"only_second\",\n",
        "    )[\"input_ids\"]\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6b32093b"
      },
      "outputs": [],
      "source": [
        "tokenized_example = tokenizer(\n",
        "    example[\"question\"],\n",
        "    example[\"context\"],\n",
        "    max_length=max_length,\n",
        "    truncation=\"only_second\",\n",
        "    return_overflowing_tokens=True,\n",
        "    stride=doc_stride,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "18176ade",
        "outputId": "8ee21c82-476e-46ad-cb7e-75450f6ac548"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[384, 157]"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "[len(x) for x in tokenized_example[\"input_ids\"]]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7dc10d93",
        "outputId": "4dabd659-2fd9-4a4c-9dce-389df593ba5b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[CLS] how many wins does the notre dame men's basketball team have? [SEP] the men's basketball team has over 1, 600 wins, one of only 12 schools who have reached that mark, and have appeared in 28 ncaa tournaments. former player austin carr holds the record for most points scored in a single game of the tournament with 61. although the team has never won the ncaa tournament, they were named by the helms athletic foundation as national champions twice. the team has orchestrated a number of upsets of number one ranked teams, the most notable of which was ending ucla's record 88 - game winning streak in 1974. the team has beaten an additional eight number - one teams, and those nine wins rank second, to ucla's 10, all - time in wins against the top team. the team plays in newly renovated purcell pavilion ( within the edmund p. joyce center ), which reopened for the beginning of the 2009 – 2010 season. the team is coached by mike brey, who, as of the 2014 – 15 season, his fifteenth at notre dame, has achieved a 332 - 165 record. in 2009 they were invited to the nit, where they advanced to the semifinals but were beaten by penn state who went on and beat baylor in the championship. the 2010 – 11 team concluded its regular season ranked number seven in the country, with a record of 25 – 5, brey's fifth straight 20 - win season, and a second - place finish in the big east. during the 2014 - 15 season, the team went 32 - 6 and won the acc conference tournament, later advancing to the elite 8, where the fighting irish lost on a missed buzzer - beater against then undefeated kentucky. led by nba draft picks jerian grant and pat connaughton, the fighting irish beat the eventual national champion duke blue devils twice during the season. the 32 wins were [SEP]\n",
            "[CLS] how many wins does the notre dame men's basketball team have? [SEP] championship. the 2010 – 11 team concluded its regular season ranked number seven in the country, with a record of 25 – 5, brey's fifth straight 20 - win season, and a second - place finish in the big east. during the 2014 - 15 season, the team went 32 - 6 and won the acc conference tournament, later advancing to the elite 8, where the fighting irish lost on a missed buzzer - beater against then undefeated kentucky. led by nba draft picks jerian grant and pat connaughton, the fighting irish beat the eventual national champion duke blue devils twice during the season. the 32 wins were the most by the fighting irish team since 1908 - 09. [SEP]\n"
          ]
        }
      ],
      "source": [
        "for x in tokenized_example[\"input_ids\"][:2]:\n",
        "    print(tokenizer.decode(x))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "45e348bc",
        "outputId": "a70ed31e-212e-4b67-bec7-2f440e250570"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[(0, 0), (0, 3), (4, 8), (9, 13), (14, 18), (19, 22), (23, 28), (29, 33), (34, 37), (37, 38), (38, 39), (40, 50), (51, 55), (56, 60), (60, 61), (0, 0), (0, 3), (4, 7), (7, 8), (8, 9), (10, 20), (21, 25), (26, 29), (30, 34), (35, 36), (36, 37), (37, 40), (41, 45), (45, 46), (47, 50), (51, 53), (54, 58), (59, 61), (62, 69), (70, 73), (74, 78), (79, 86), (87, 91), (92, 96), (96, 97), (98, 101), (102, 106), (107, 115), (116, 118), (119, 121), (122, 126), (127, 138), (138, 139), (140, 146), (147, 153), (154, 160), (161, 165), (166, 171), (172, 175), (176, 182), (183, 186), (187, 191), (192, 198), (199, 205), (206, 208), (209, 210), (211, 217), (218, 222), (223, 225), (226, 229), (230, 240), (241, 245), (246, 248), (248, 249), (250, 258), (259, 262), (263, 267), (268, 271), (272, 277), (278, 281), (282, 285), (286, 290), (291, 301), (301, 302), (303, 307), (308, 312), (313, 318), (319, 321), (322, 325), (326, 330), (330, 331), (332, 340), (341, 351), (352, 354), (355, 363), (364, 373), (374, 379), (379, 380), (381, 384), (385, 389), (390, 393), (394, 406), (407, 408), (409, 415), (416, 418)]\n"
          ]
        }
      ],
      "source": [
        "tokenized_example = tokenizer(\n",
        "    example[\"question\"],\n",
        "    example[\"context\"],\n",
        "    max_length=max_length,\n",
        "    truncation=\"only_second\",\n",
        "    return_overflowing_tokens=True,\n",
        "    return_offsets_mapping=True,\n",
        "    stride=doc_stride,\n",
        ")\n",
        "print(tokenized_example[\"offset_mapping\"][0][:100])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e2dcd8a5",
        "outputId": "a4ada182-cbb7-46a8-b15a-6d4fd4da55d1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "how How\n"
          ]
        }
      ],
      "source": [
        "first_token_id = tokenized_example[\"input_ids\"][0][1]\n",
        "offsets = tokenized_example[\"offset_mapping\"][0][1]\n",
        "print(\n",
        "    tokenizer.convert_ids_to_tokens([first_token_id])[0],\n",
        "    example[\"question\"][offsets[0] : offsets[1]],\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "079cdafe",
        "outputId": "fd7e9a7e-63ab-4861-a9ce-c5cba5735909"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[None, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, None, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, None]\n"
          ]
        }
      ],
      "source": [
        "sequence_ids = tokenized_example.sequence_ids()\n",
        "print(sequence_ids)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2d890f25",
        "outputId": "4a346f97-b2dc-4a4c-9cf8-6197c09782a7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "23 26\n"
          ]
        }
      ],
      "source": [
        "answers = example[\"answers\"]\n",
        "start_char = answers[\"answer_start\"][0]\n",
        "end_char = start_char + len(answers[\"text\"][0])\n",
        "\n",
        "# Start token index of the current span in the text.\n",
        "token_start_index = 0\n",
        "while sequence_ids[token_start_index] != 1:\n",
        "    token_start_index += 1\n",
        "\n",
        "# End token index of the current span in the text.\n",
        "token_end_index = len(tokenized_example[\"input_ids\"][0]) - 1\n",
        "while sequence_ids[token_end_index] != 1:\n",
        "    token_end_index -= 1\n",
        "\n",
        "# Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).\n",
        "offsets = tokenized_example[\"offset_mapping\"][0]\n",
        "if (\n",
        "    offsets[token_start_index][0] <= start_char\n",
        "    and offsets[token_end_index][1] >= end_char\n",
        "):\n",
        "    # Move the token_start_index and token_end_index to the two ends of the answer.\n",
        "    # Note: we could go after the last offset if the answer is the last word (edge case).\n",
        "    while (\n",
        "        token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char\n",
        "    ):\n",
        "        token_start_index += 1\n",
        "    start_position = token_start_index - 1\n",
        "    while offsets[token_end_index][1] >= end_char:\n",
        "        token_end_index -= 1\n",
        "    end_position = token_end_index + 1\n",
        "    print(start_position, end_position)\n",
        "else:\n",
        "    print(\"The answer is not in this feature.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b5ffe48a",
        "outputId": "27165f61-55c6-4707-aab6-efd5c6fb5637"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "over 1, 600\n",
            "over 1,600\n"
          ]
        }
      ],
      "source": [
        "print(\n",
        "    tokenizer.decode(\n",
        "        tokenized_example[\"input_ids\"][0][start_position : end_position + 1]\n",
        "    )\n",
        ")\n",
        "print(answers[\"text\"][0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bb84fce7"
      },
      "outputs": [],
      "source": [
        "pad_on_right = tokenizer.padding_side == \"right\"\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 1.3 Prepare for Training "
      ],
      "metadata": {
        "id": "b0muc525MIrh"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8ca68804"
      },
      "outputs": [],
      "source": [
        "def prepare_train_features(examples):\n",
        "    # Tokenize our examples with truncation and padding, but keep the overflows using a stride. This results\n",
        "    # in one example possible giving several features when a context is long, each of those features having a\n",
        "    # context that overlaps a bit the context of the previous feature.\n",
        "    tokenized_examples = tokenizer(\n",
        "        examples[\"question\" if pad_on_right else \"context\"],\n",
        "        examples[\"context\" if pad_on_right else \"question\"],\n",
        "        truncation=\"only_second\" if pad_on_right else \"only_first\",\n",
        "        max_length=max_length,\n",
        "        stride=doc_stride,\n",
        "        return_overflowing_tokens=True,\n",
        "        return_offsets_mapping=True,\n",
        "        padding=\"max_length\",\n",
        "    )\n",
        "\n",
        "    # Since one example might give us several features if it has a long context, we need a map from a feature to\n",
        "    # its corresponding example. This key gives us just that.\n",
        "    sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n",
        "    # The offset mappings will give us a map from token to character position in the original context. This will\n",
        "    # help us compute the start_positions and end_positions.\n",
        "    offset_mapping = tokenized_examples.pop(\"offset_mapping\")\n",
        "\n",
        "    # Let's label those examples!\n",
        "    tokenized_examples[\"start_positions\"] = []\n",
        "    tokenized_examples[\"end_positions\"] = []\n",
        "\n",
        "    for i, offsets in enumerate(offset_mapping):\n",
        "        # We will label impossible answers with the index of the CLS token.\n",
        "        input_ids = tokenized_examples[\"input_ids\"][i]\n",
        "        cls_index = input_ids.index(tokenizer.cls_token_id)\n",
        "\n",
        "        # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n",
        "        sequence_ids = tokenized_examples.sequence_ids(i)\n",
        "\n",
        "        # One example can give several spans, this is the index of the example containing this span of text.\n",
        "        sample_index = sample_mapping[i]\n",
        "        answers = examples[\"answers\"][sample_index]\n",
        "        # If no answers are given, set the cls_index as answer.\n",
        "        if len(answers[\"answer_start\"]) == 0:\n",
        "            tokenized_examples[\"start_positions\"].append(cls_index)\n",
        "            tokenized_examples[\"end_positions\"].append(cls_index)\n",
        "        else:\n",
        "            # Start/end character index of the answer in the text.\n",
        "            start_char = answers[\"answer_start\"][0]\n",
        "            end_char = start_char + len(answers[\"text\"][0])\n",
        "\n",
        "            # Start token index of the current span in the text.\n",
        "            token_start_index = 0\n",
        "            while sequence_ids[token_start_index] != (1 if pad_on_right else 0):\n",
        "                token_start_index += 1\n",
        "\n",
        "            # End token index of the current span in the text.\n",
        "            token_end_index = len(input_ids) - 1\n",
        "            while sequence_ids[token_end_index] != (1 if pad_on_right else 0):\n",
        "                token_end_index -= 1\n",
        "\n",
        "            # Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).\n",
        "            if not (\n",
        "                offsets[token_start_index][0] <= start_char\n",
        "                and offsets[token_end_index][1] >= end_char\n",
        "            ):\n",
        "                tokenized_examples[\"start_positions\"].append(cls_index)\n",
        "                tokenized_examples[\"end_positions\"].append(cls_index)\n",
        "            else:\n",
        "                # Otherwise move the token_start_index and token_end_index to the two ends of the answer.\n",
        "                # Note: we could go after the last offset if the answer is the last word (edge case).\n",
        "                while (\n",
        "                    token_start_index < len(offsets)\n",
        "                    and offsets[token_start_index][0] <= start_char\n",
        "                ):\n",
        "                    token_start_index += 1\n",
        "                tokenized_examples[\"start_positions\"].append(token_start_index - 1)\n",
        "                while offsets[token_end_index][1] >= end_char:\n",
        "                    token_end_index -= 1\n",
        "                tokenized_examples[\"end_positions\"].append(token_end_index + 1)\n",
        "\n",
        "    return tokenized_examples"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1239c816"
      },
      "outputs": [],
      "source": [
        "features = prepare_train_features(datasets[\"train\"][:5])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "021726d9",
        "outputId": "2028bdbc-c240-4596-fe9d-816fc0153ca7"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Loading cached processed dataset at /Users/hs/.cache/huggingface/datasets/squad/plain_text/1.0.0/d6ec3ceb99ca480ce37cdd35555d6cb2511d223b9150cce08a837ef62ffea453/cache-c0850278904a0778.arrow\n",
            "Loading cached processed dataset at /Users/hs/.cache/huggingface/datasets/squad/plain_text/1.0.0/d6ec3ceb99ca480ce37cdd35555d6cb2511d223b9150cce08a837ef62ffea453/cache-c15cf83fb23e354d.arrow\n"
          ]
        }
      ],
      "source": [
        "tokenized_datasets = datasets.map(\n",
        "    prepare_train_features, batched=True, remove_columns=datasets[\"train\"].column_names\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2db0c7b2",
        "outputId": "9a8d4927-9203-44e8-ad9a-c6adcd6752b2"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2022-08-13 21:09:40.108495: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
            "2022-08-13 21:09:40.109059: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Metal device set to: Apple M1\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertForQuestionAnswering: ['vocab_layer_norm', 'vocab_projector', 'vocab_transform', 'activation_13']\n",
            "- This IS expected if you are initializing TFDistilBertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFDistilBertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some layers of TFDistilBertForQuestionAnswering were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['qa_outputs', 'dropout_19']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "from transformers import TFAutoModelForQuestionAnswering\n",
        "\n",
        "model = TFAutoModelForQuestionAnswering.from_pretrained(model_checkpoint)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 1.4 Train the model"
      ],
      "metadata": {
        "id": "ku12bptEMWUt"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a9ee8b59"
      },
      "outputs": [],
      "source": [
        "model_name = model_checkpoint.split(\"/\")[-1]\n",
        "push_to_hub_model_id = f\"{model_name}-finetuned-squad\"\n",
        "learning_rate = 2e-5\n",
        "num_train_epochs = 2\n",
        "weight_decay = 0.01"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b52735c1"
      },
      "outputs": [],
      "source": [
        "from transformers import DefaultDataCollator\n",
        "\n",
        "data_collator = DefaultDataCollator(return_tensors=\"tf\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f9cee1e5"
      },
      "outputs": [],
      "source": [
        "train_set = tokenized_datasets[\"train\"].to_tf_dataset(\n",
        "    columns=[\"attention_mask\", \"input_ids\", \"start_positions\", \"end_positions\"],\n",
        "    shuffle=True,\n",
        "    batch_size=batch_size,\n",
        "    collate_fn=data_collator,\n",
        ")\n",
        "validation_set = tokenized_datasets[\"validation\"].to_tf_dataset(\n",
        "    columns=[\"attention_mask\", \"input_ids\", \"start_positions\", \"end_positions\"],\n",
        "    shuffle=False,\n",
        "    batch_size=batch_size,\n",
        "    collate_fn=data_collator,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e8852900"
      },
      "outputs": [],
      "source": [
        "from transformers import create_optimizer\n",
        "\n",
        "total_train_steps = (len(tokenized_datasets[\"train\"]) // batch_size) * num_train_epochs\n",
        "\n",
        "optimizer, schedule = create_optimizer(\n",
        "    init_lr=learning_rate, num_warmup_steps=0, num_train_steps=total_train_steps\n",
        ")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "c1b4db18",
        "outputId": "8072abb7-5c79-4779-c258-7ef45f86058b"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "No loss specified in compile() - the model's internal loss computation will be used as the loss. Don't panic - this is a common way to train TensorFlow models in Transformers! To disable this behaviour please pass a loss argument, or explicitly pass `loss=None` if you do not want your model to compute a loss.\n"
          ]
        }
      ],
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "model.compile(optimizer=optimizer, metrics=[\"accuracy\"])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "outputId": "ccb1c085-eede-4fe0-914e-0754fca7ceac",
        "id": "0Z5nG68nNUeR"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"tf_distil_bert_for_question_answering\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " distilbert (TFDistilBertMai  multiple                 66362880  \n",
            " nLayer)                                                         \n",
            "                                                                 \n",
            " qa_outputs (Dense)          multiple                  1538      \n",
            "                                                                 \n",
            " dropout_19 (Dropout)        multiple                  0         \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 66,364,418\n",
            "Trainable params: 66,364,418\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F5wGx6bJ-BJC"
      },
      "source": [
        "def read_data(path):  \n",
        "  # load the json file\n",
        "  with open(path, 'rb') as f:\n",
        "    squad = json.load(f)\n",
        "\n",
        "  contexts = []\n",
        "  questions = []\n",
        "  answers = []\n",
        "\n",
        "  for group in squad['data']:\n",
        "    for passage in group['paragraphs']:\n",
        "      context = passage['context']\n",
        "      for qa in passage['qas']:\n",
        "        question = qa['question']\n",
        "        for answer in qa['answers']:\n",
        "          contexts.append(context)\n",
        "          questions.append(question)\n",
        "          answers.append(answer)\n",
        "\n",
        "  return contexts, questions, answers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "An5wY1MNEwrt"
      },
      "source": [
        "train_contexts, train_questions, train_answers = read_data('train-v2.0.json')\n",
        "valid_contexts, valid_questions, valid_answers = read_data('dev-v2.0.json')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oER4NnGvFfDG",
        "outputId": "78994f3d-a069-4c96-a738-9643d5681490"
      },
      "source": [
        "# print a random question and answer\n",
        "print(f'There are {len(train_questions)} questions')\n",
        "print(train_questions[-10000])\n",
        "print(train_answers[-10000])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are 86821 questions\n",
            "What is a modern common occurence with antibiotics?\n",
            "{'text': 'resistance of bacteria', 'answer_start': 17}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b9kiFd7LF6xa"
      },
      "source": [
        "def add_end_idx(answers, contexts):\n",
        "  for answer, context in zip(answers, contexts):\n",
        "    gold_text = answer['text']\n",
        "    start_idx = answer['answer_start']\n",
        "    end_idx = start_idx + len(gold_text)\n",
        "\n",
        "    # sometimes squad answers are off by a character or two so we fix this\n",
        "    if context[start_idx:end_idx] == gold_text:\n",
        "      answer['answer_end'] = end_idx\n",
        "    elif context[start_idx-1:end_idx-1] == gold_text:\n",
        "      answer['answer_start'] = start_idx - 1\n",
        "      answer['answer_end'] = end_idx - 1     # When the gold label is off by one character\n",
        "    elif context[start_idx-2:end_idx-2] == gold_text:\n",
        "      answer['answer_start'] = start_idx - 2\n",
        "      answer['answer_end'] = end_idx - 2     # When the gold label is off by two characters\n",
        "\n",
        "add_end_idx(train_answers, train_contexts)\n",
        "add_end_idx(valid_answers, valid_contexts)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "baV-xG3QHGPR",
        "outputId": "083bd0a6-7d33-4e8e-c7eb-cdaaa45e069c"
      },
      "source": [
        "# You can see that now we get the answer_end also\n",
        "print(train_questions[-10000])\n",
        "print(train_answers[-10000])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "What is a modern common occurence with antibiotics?\n",
            "{'text': 'resistance of bacteria', 'answer_start': 17, 'answer_end': 39}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 174
        },
        "id": "D8nEGmjZMIaW",
        "outputId": "a5566afc-48fe-49c0-bc3a-e0c57de16d10"
      },
      "source": [
        "tokenizer.decode(train_encodings['input_ids'][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'[CLS] beyonce giselle knowles - carter ( / biːˈjɒnseɪ / bee - yon - say ) ( born september 4, 1981 ) is an american singer, songwriter, record producer and actress. born and raised in houston, texas, she performed in various singing and dancing competitions as a child, and rose to fame in the late 1990s as lead singer of r & b girl - group destiny\\'s child. managed by her father, mathew knowles, the group became one of the world\\'s best - selling girl groups of all time. their hiatus saw the release of beyonce\\'s debut album, dangerously in love ( 2003 ), which established her as a solo artist worldwide, earned five grammy awards and featured the billboard hot 100 number - one singles \" crazy in love \" and \" baby boy \". [SEP] when did beyonce start becoming popular? [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]'"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r7uXDM0mMfOo"
      },
      "source": [
        "def add_token_positions(encodings, answers):\n",
        "  start_positions = []\n",
        "  end_positions = []\n",
        "  for i in range(len(answers)):\n",
        "    start_positions.append(encodings.char_to_token(i, answers[i]['answer_start']))\n",
        "    end_positions.append(encodings.char_to_token(i, answers[i]['answer_end'] - 1))\n",
        "\n",
        "    # if start position is None, the answer passage has been truncated\n",
        "    if start_positions[-1] is None:\n",
        "      start_positions[-1] = tokenizer.model_max_length\n",
        "    if end_positions[-1] is None:\n",
        "      end_positions[-1] = tokenizer.model_max_length\n",
        "\n",
        "  encodings.update({'start_positions': start_positions, 'end_positions': end_positions})\n",
        "\n",
        "add_token_positions(train_encodings, train_answers)\n",
        "add_token_positions(valid_encodings, valid_answers)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cU-mZO2RMwVl",
        "outputId": "cf33e5a9-63dc-4581-f5e8-ba2e71bf723d"
      },
      "source": [
        "train_encodings['start_positions'][:10]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[67, 55, 128, 47, 69, 81, 124, 91, 69, 72]"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 1.5 Setting evaluation metrics"
      ],
      "metadata": {
        "id": "GR3PYnd-ObsG"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JZd-NV1WxOZg"
      },
      "source": [
        "model.eval()\n",
        "\n",
        "acc = []\n",
        "\n",
        "for batch in tqdm(valid_loader):\n",
        "  with torch.no_grad():\n",
        "    input_ids = batch['input_ids'].to(device)\n",
        "    attention_mask = batch['attention_mask'].to(device)\n",
        "    start_true = batch['start_positions'].to(device)\n",
        "    end_true = batch['end_positions'].to(device)\n",
        "    \n",
        "    outputs = model(input_ids, attention_mask=attention_mask)\n",
        "\n",
        "    start_pred = torch.argmax(outputs['start_logits'], dim=1)\n",
        "    end_pred = torch.argmax(outputs['end_logits'], dim=1)\n",
        "\n",
        "    acc.append(((start_pred == start_true).sum()/len(start_pred)).item())\n",
        "    acc.append(((end_pred == end_true).sum()/len(end_pred)).item())\n",
        "\n",
        "acc = sum(acc)/len(acc)\n",
        "\n",
        "print(\"\\n\\nT/P\\tanswer_start\\tanswer_end\\n\")\n",
        "for i in range(len(start_true)):\n",
        "  print(f\"true\\t{start_true[i]}\\t{end_true[i]}\\n\"\n",
        "        f\"pred\\t{start_pred[i]}\\t{end_pred[i]}\\n\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d5oTgk-wSvyL"
      },
      "source": [
        "def get_prediction(context, question):\n",
        "  inputs = tokenizer.encode_plus(question, context, return_tensors='pt').to(device)\n",
        "  outputs = model(**inputs)\n",
        "  \n",
        "  answer_start = torch.argmax(outputs[0])  \n",
        "  answer_end = torch.argmax(outputs[1]) + 1 \n",
        "  \n",
        "  answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(inputs['input_ids'][0][answer_start:answer_end]))\n",
        "  \n",
        "  return answer\n",
        "\n",
        "def normalize_text(s):\n",
        "  \"\"\"Removing articles and punctuation, and standardizing whitespace are all typical text processing steps.\"\"\"\n",
        "  import string, re\n",
        "  def remove_articles(text):\n",
        "    regex = re.compile(r\"\\b(a|an|the)\\b\", re.UNICODE)\n",
        "    return re.sub(regex, \" \", text)\n",
        "  def white_space_fix(text):\n",
        "    return \" \".join(text.split())\n",
        "  def remove_punc(text):\n",
        "    exclude = set(string.punctuation)\n",
        "    return \"\".join(ch for ch in text if ch not in exclude)\n",
        "  def lower(text):\n",
        "    return text.lower()\n",
        "\n",
        "  return white_space_fix(remove_articles(remove_punc(lower(s))))\n",
        "\n",
        "def exact_match(prediction, truth):\n",
        "    return bool(normalize_text(prediction) == normalize_text(truth))\n",
        "\n",
        "def compute_f1(prediction, truth):\n",
        "  pred_tokens = normalize_text(prediction).split()\n",
        "  truth_tokens = normalize_text(truth).split()\n",
        "  \n",
        "  # if either the prediction or the truth is no-answer then f1 = 1 if they agree, 0 otherwise\n",
        "  if len(pred_tokens) == 0 or len(truth_tokens) == 0:\n",
        "    return int(pred_tokens == truth_tokens)\n",
        "  \n",
        "  common_tokens = set(pred_tokens) & set(truth_tokens)\n",
        "  \n",
        "  # if there are no common tokens then f1 = 0\n",
        "  if len(common_tokens) == 0:\n",
        "    return 0\n",
        "  \n",
        "  prec = len(common_tokens) / len(pred_tokens)\n",
        "  rec = len(common_tokens) / len(truth_tokens)\n",
        "  \n",
        "  return round(2 * (prec * rec) / (prec + rec), 2)\n",
        "  \n",
        "def question_answer(context, question,answer):\n",
        "  prediction = get_prediction(context,question)\n",
        "  em_score = exact_match(prediction, answer)\n",
        "  f1_score = compute_f1(prediction, answer)\n",
        "\n",
        "  print(f'Question: {question}')\n",
        "  print(f'Prediction: {prediction}')\n",
        "  print(f'True Answer: {answer}')\n",
        "  print(f'Exact match: {em_score}')\n",
        "  print(f'F1 score: {f1_score}\\n')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "clBBH3gWLMjI"
      },
      "source": [
        "## 1.6 Verification QnA model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JBos0I6qL1ht",
        "outputId": "4d9331c8-b302-4318-9efc-b2f9a6c49b58"
      },
      "source": [
        "context = \"\"\"Athens is the capital and largest city of Greece. Athens dominates the Attica region and is one of the world's oldest cities, \n",
        "             with its recorded history spanning over 3,400 years and its earliest human presence starting somewhere between the 11th and 7th millennium BC.\n",
        "             Classical Athens was a powerful city-state. It was a center for the arts, learning and philosophy, and the home of Plato's Academy and Aristotle's Lyceum.\n",
        "             It is widely referred to as the cradle of Western civilization and the birthplace of democracy, largely because of its cultural and political impact on the European continent—particularly Ancient Rome.\n",
        "             In modern times, Athens is a large cosmopolitan metropolis and central to economic, financial, industrial, maritime, political and cultural life in Greece. \n",
        "             In 2021, Athens' urban area hosted more than three and a half million people, which is around 35% of the entire population of Greece.\n",
        "             Athens is a Beta global city according to the Globalization and World Cities Research Network, and is one of the biggest economic centers in Southeastern Europe. \n",
        "             It also has a large financial sector, and its port Piraeus is both the largest passenger port in Europe, and the second largest in the world.\"\"\"\n",
        "\n",
        "questions = [\"Which is the largest city in Greece?\",\n",
        "             \"For what was the Athens center?\",\n",
        "             \"Which city was the home of Plato's Academy?\"]\n",
        "\n",
        "answers = [\"Athens\", \"center for the arts, learning and philosophy\", \"Athens\"]\n",
        "\n",
        "for question, answer in zip(questions, answers):\n",
        "  question_answer(context, question, answer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: Which is the largest city in Greece?\n",
            "Prediction: athens\n",
            "True Answer: Athens\n",
            "Exact match: True\n",
            "F1 score: 1.0\n",
            "\n",
            "Question: For what was the Athens center?\n",
            "Prediction: center for the arts, learning and philosophy,\n",
            "True Answer: center for the arts, learning and philosophy\n",
            "Exact match: True\n",
            "F1 score: 1.0\n",
            "\n",
            "Question: Which city was the home of Plato's Academy?\n",
            "Prediction: athens\n",
            "True Answer: Athens\n",
            "Exact match: True\n",
            "F1 score: 1.0\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 2. Translation "
      ],
      "metadata": {
        "id": "wszDJLPU_tGk"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 2.1 Import modules"
      ],
      "metadata": {
        "id": "ZyooEsjmAK2J"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AxL9VC6EWlNb"
      },
      "outputs": [],
      "source": [
        "import string\n",
        "import re\n",
        "from numpy import array, argmax, random, take\n",
        "import pandas as pd\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, LSTM, Embedding, RepeatVector\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "from keras_preprocessing.sequence import pad_sequences\n",
        "from keras.models import load_model\n",
        "from keras import optimizers\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "pd.set_option('display.max_colwidth', 200)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 2.2 Load Dataset and Preprocessing"
      ],
      "metadata": {
        "id": "dHGJip1LATUo"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NpSiSvRwXTVQ"
      },
      "outputs": [],
      "source": [
        "# function to read raw text file\n",
        "def read_text(filename):\n",
        "        # open the file\n",
        "        file = open(filename, mode='rt', encoding='utf-8')\n",
        "        \n",
        "        # read all text\n",
        "        text = file.read()\n",
        "        file.close()\n",
        "        return text\n",
        "# Let’s define another function to split the text into English-German pairs separated by ‘\\n’. We’ll then split these pairs into English sentences and German sentences respectively.\n",
        "# split a text into sentences\n",
        "\n",
        "def to_lines(text):\n",
        "      sents = text.strip().split('\\n')\n",
        "      sents = [i.split('\\t') for i in sents]\n",
        "      for i in sents:\n",
        "        i.pop()\n",
        "      return sents\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8lKJQ24XXZkV"
      },
      "outputs": [],
      "source": [
        "#We can now use these functions to read the text into an array in our desired format.\n",
        "\n",
        "data = read_text(\"deu.txt\")\n",
        "deu_eng = to_lines(data)\n",
        "deu_eng = array(deu_eng)\n",
        "deu_eng = deu_eng[:50000,:]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tzfI72bJXa1z",
        "outputId": "88f9f8b0-fc21-4211-9fae-77b92aca5491"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([['Go.', 'Geh.'],\n",
              "       ['Hi.', 'Hallo!'],\n",
              "       ['Hi.', 'Grüß Gott!'],\n",
              "       ...,\n",
              "       ['Tom has no free time.', 'Tom hat keine Freizeit.'],\n",
              "       ['Tom has no neighbors.', 'Tom hat keine Nachbarn.'],\n",
              "       ['Tom has nothing left.', 'Tom hat nichts mehr übrig.']],\n",
              "      dtype='<U537')"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "deu_eng"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EE63GMjSZbDp",
        "outputId": "77c4566e-7eeb-4aa0-ef6f-21ad03a0625b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([['Go', 'Geh'],\n",
              "       ['Hi', 'Hallo'],\n",
              "       ['Hi', 'Grüß Gott'],\n",
              "       ...,\n",
              "       ['Tom has no free time', 'Tom hat keine Freizeit'],\n",
              "       ['Tom has no neighbors', 'Tom hat keine Nachbarn'],\n",
              "       ['Tom has nothing left', 'Tom hat nichts mehr übrig']],\n",
              "      dtype='<U537')"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Remove punctuation\n",
        "deu_eng[:,0] = [s.translate(str.maketrans('', '', string.punctuation)) for s in deu_eng[:,0]]\n",
        "deu_eng[:,1] = [s.translate(str.maketrans('', '', string.punctuation)) for s in deu_eng[:,1]]\n",
        "\n",
        "deu_eng"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Alwd0WJMZfEJ",
        "outputId": "801a9ad2-fc22-47f0-eb02-f4a7b9245d32"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([['go', 'geh'],\n",
              "       ['hi', 'hallo'],\n",
              "       ['hi', 'grüß gott'],\n",
              "       ...,\n",
              "       ['tom has no free time', 'tom hat keine freizeit'],\n",
              "       ['tom has no neighbors', 'tom hat keine nachbarn'],\n",
              "       ['tom has nothing left', 'tom hat nichts mehr übrig']],\n",
              "      dtype='<U537')"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# convert text to lowercase\n",
        "for i in range(len(deu_eng)):\n",
        "    deu_eng[i,0] = deu_eng[i,0].lower()\n",
        "    deu_eng[i,1] = deu_eng[i,1].lower()\n",
        "\n",
        "deu_eng"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "id": "l4QSOnclZjld",
        "outputId": "4a6b3dd4-bbf4-4b6c-fa9a-01738a94dd19"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEICAYAAAC9E5gJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAbsElEQVR4nO3df5Ac5X3n8ffHEhAF2wEMtwFJ9iqJzBUgW8AGcUXOWYcYxI9Y4HMREQKSzSEooxjuVBULn6ugwFwpP2QHfD5sgRWJO4JQDBgFZGRF8RZ2nQUSoGP5YU6LECWphGQj8UOQwln8vT/6WdSanZ2dmZ2dntn9vKqmpufpp7u/PerVt/vpZ/pRRGBmZuPbB4oOwMzMiudkYGZmTgZmZuZkYGZmOBmYmRlOBmZmhpOBmbUhSSskfb3oOMYSJwMzM3MyMDMzJ4O2JOkESfdL+oWklyV9OZXfJGm1pLslvSXpOUldueVOk/R0mvePku7zpba1A0mnSnoqHbv3Ab+Rm3ehpC2SXpf0fyR9IjcvJP1e7rObl4bgZNBmJH0A+Cfg/wKTgbOB6yWdm6p8FlgFHAWsAf5HWu5w4EFgBXAMcC9wcRNDN6tLOnZ/APwvsmP3H4H/lOadCiwHrgY+AnwXWCPpiEKCbWNOBu3n94HjIuLmiPhVRGwD7gTmpvk/jYi1EfEe2R/PJ1P5mcBE4PaI+LeIeAB4otnBm9XhTOAw4O/Ssft9YFOatwD4bkQ8HhHvRcRK4N20jNVgYtEBWM0+Bpwg6fVc2QTgJ8ArwKu58neA35A0ETgB2BWHPplwxyjHatYI5Y7dV9L7x4B5kv4iN+/wtIzVwFcG7WcH8HJEHJV7fSgizh9mud3AZEnKlU0dvTDNGqbcsfvR9L4DuLXk7+E3I+LeNP8d4Ddzy/12E+JtS04G7ecJ4C1JX5E0SdIESadI+v1hlvsZ8B6wUNJESXOAM0Y9WrOR+xnQD3xZ0mGSPsfBY/dO4BpJs5Q5UtIFkj6U5m8B/iz9ncwG/rDp0bcJJ4M2k+4FXAjMBF4GfgncBfzWMMv9CvgccCXwOvDnwMNk7atmLSt37M4H9gF/CjyQ5m0GriLrKLEf6Ev1BlwH/AnZMX8Z2Y1oK0Me3Gb8kvQ48J2I+PuiYzGzYvnKYByR9IeSfjs1E80DPgE8WnRcZlY89yYaX04EVgNHAtuAz0fE7mJDMrNW4GYiMzNzM5GZmbVxM9Gxxx4bnZ2dhcbw9ttvc+SRRxYaQ6ONp3168sknfxkRxxUQUl1a4ZivRbsfS+0cf6XYhzru2zYZdHZ2snnz5kJj6Onpobu7u9AYGm087ZOkVwbXbl2tcMzXot2PpXaOv1LsQx33biYyMzMnAzMzczIwMzOcDMzMDCcDMzPDycDMzHAyMDMznAzMzAwnAzMzo41/gWyjo3fXG8xf/Mj7n7cvuaDAaKzVdOaODfDxMZb4ysDMzJwMzMzMycDMzHAyMBtE0lRJP5b0vKTnJF2Xyo+RtF7S1vR+dCqXpNsl9Ul6RtJpuXXNS/W3pqFGB8pPl9Sblrldkpq/p2YHORmYDdYPLIqIk4AzgWslnQQsBjZExHRgQ/oMcB4wPb0WAHdAljyAG4FZwBnAjQMJJNW5Krfc7Cbsl9mQnAzMSkTE7oh4Kk2/BbwATAbmACtTtZXARWl6DnB3ZDYCR0k6HjgXWB8R+yJiP7AemJ3mfTgiNkY27uzduXWZFcJdS80qkNQJnAo8DnRExO4061WgI01PBnbkFtuZyiqV7yxTXm77C8iuNujo6KCnp6f+nWmARTP6D/lcKZ4DBw4UHu9ItHP89cTuZGA2BEkfBO4Hro+IN/PN+hERkmK0Y4iIZcAygK6urih65K35pb8zuKx7yLrtPFIYtHf89cTuZiKzMiQdRpYI7omIB1LxntTEQ3rfm8p3AVNzi09JZZXKp5QpNyuMk4FZidSz53vACxHxjdysNcBAj6B5wEO58itSr6IzgTdSc9I64BxJR6cbx+cA69K8NyWdmbZ1RW5dZoVwM5HZYGcBlwO9kraksq8CS4DVkq4EXgEuSfPWAucDfcA7wBcAImKfpFuATanezRGxL01/CVgBTAJ+mF5mhRk2GUiaStbboQMIYFlE3Ja6zd0HdALbgUsiYn8607mN7I/jHWD+QM+M1M/6a2nVX4+Ilan8dA7+YawFrku9LMyaLiJ+CgzV7//sMvUDuHaIdS0Hlpcp3wycMoIwzRqqmmYi97k2Mxvjhk0G7nNtZjb21XQDueg+12ZmNjqqvoHcCn2uW+0HOO38o5ShdEw69IdFY2H/xuK/k1mjVZUMKvW5jojdNfS57i4p76GGPtet9gOcdv5RylC+dc9DLO09eFhU+lFRuxiL/05mjTZsM5H7XJuZjX3VXBm4z7WZ2Rg3bDJwn2szs7HPj6MwMzMnAzMzczIwMzOcDMzMDCcDMzPDycDMzHAyMDMznAzMzAwnA7NBJC2XtFfSs7my+yRtSa/tA7/Gl9Qp6V9z876TW+Z0Sb2S+iTdnh63gqRjJK2XtDW9Hz0oCLMmczIwG2wFJQMsRcSfRsTMiJhJ9tDGB3KzXxqYFxHX5MqHGrRpqIGhzArjZGBWIiIeA/aVm5fO7i8B7q20jmEGbRpqYCizwlQ9noGZAfAfgT0RsTVXNk3S08CbwNci4idUHrRpqIGhBmm1MTzyY11A5fEu2n0ciXaOv57YnQzManMph14V7AY+GhGvSTod+IGkk6td2XADQ7XaGB7zFz9yyOdK4120+zgS7Rx/PbE7GZhVSdJE4HPA6QNlEfEu8G6aflLSS8DHqTxo01ADQ5kVxvcMzKr3x8DPI+L95h9Jx0makKZ/h+xG8bZhBm0aamAos8I4GZiVkHQv8DPgREk70wBOAHMZfOP4U8Azqavp94FrSgZtuotsoKeXODho0xLgM5K2kiWYJaO1L2bVcjORWYmIuHSI8vllyu4n62parn7ZQZsi4jXKDAxlViRfGZiZmZOBmZk5GZiZGU4GZmaGk4GZmeHeRG2hs+RXnwDbl1xQQCRmNlb5ysDMzJwMzMzMycDMzHAyMDMznAzMzAwnAzMzw8nAzMxwMjAzM5wMzMwMJwMzM8PJwKwsScsl7ZX0bK7sJkm7JG1Jr/Nz826Q1CfpRUnn5spnp7I+SYtz5dMkPZ7K75N0ePP2zmwwJwOz8lYAs8uUfzMiZqbXWgBJJ5ENiXlyWuZ/SpqQxkb+NnAecBJwaaoL8FdpXb8H7AeuLN2QWTM5GZiVERGPAfuGrZiZA6yKiHcj4mWyMY/PSK++iNgWEb8CVgFzJAn4I7IxkwFWAhc1Mn6zWvmppWa1WSjpCmAzsCgi9gOTgY25OjtTGcCOkvJZwEeA1yOiv0z9Q0haACwA6OjooKenp0G7UZ9FM/oP+VwpngMHDhQe70i0c/z1xO5kYFa9O4BbgEjvS4EvjuYGI2IZsAygq6sruru7R3Nzw5pf8jj17Zd1D1m3p6eHouMdiXaOv57YnQzMqhQRewamJd0JPJw+7gKm5qpOSWUMUf4acJSkienqIF/frBDD3jNwrwqzjKTjcx8vBgb+JtYAcyUdIWkaMB14AtgETE/H+OFkN5nXREQAPwY+n5afBzzUjH0wG0o1N5BX4F4VNs5Iuhf4GXCipJ2SrgT+WlKvpGeATwP/BSAingNWA88DjwLXRsR76ax/IbAOeAFYneoCfAX4r5L6yO4hfK+Ju2c2yLDNRBHxmKTOKtf3fq8K4OV0oJ+R5vVFxDYASQO9Kl4g61XxZ6nOSuAmsrZZs8JExKVliof8DzsibgVuLVO+FlhbpnwbB/82zAo3knsGTe1VAa3Xs6JZvQ1Ke3BA5V4cI9Ex6dDtFf0dN0I79woxa5Z6k0HTe1VA6/WsaFZvg9IeHFC5F8dIfOueh1jae/CwGK3tNFM79woxa5a6koF7VZiZjS11/QLZvSrMzMaWYa8MUq+KbuBYSTuBG4FuSTPJmom2A1dD1qtC0kCvin5Sr4q0noFeFROA5SW9KlZJ+jrwNO5VYWbWdNX0JnKvCjOzMc4PqjMzMycDMzNzMjAzM5wMzMwMJwMzM8PJwMzMcDIwMzOcDMzMDI90ZmZJZ7kHIi65oIBIrAi+MjAzMycDs1JDDPX6N5J+LukZSQ9KOiqVd0r619wQsN/JLXN6GhmtT9LtkpTKj5G0XtLW9H5003fSrISTgdlgKxg81Ot64JSI+ATw/4AbcvNeyg0Be02u/A7gKrKn907PrXMxsCEipgMb0mezQjkZmJWIiMeAfSVlP8qNyLeRbOyNIaXHvH84IjamR7XfDVyUZs8hG+KV9H7RoBWYNZlvIJvV7ovAfbnP0yQ9DbwJfC0ifkI2fOvOXJ38kK4dEbE7Tb8KdAy1oWYO9VrN8KqldSrF0+7DjbZz/PXE7mRgVgNJ/41srI57UtFu4KMR8Zqk04EfSDq52vVFREiKCvObNtRrNcOrltapNCxquw832s7x1xO7k4FZlSTNBy4Ezk5NP0TEu8C7afpJSS8BHycbvjXflJQf0nWPpOMjYndqTtrbpF0wG5LvGZhVQdJs4C+Bz0bEO7ny4yRNSNO/Q3ajeFtqBnpT0pmpF9EVHBzSdQ3ZEK/goV6tRfjKwKzEEEO93gAcAaxPPUQ3pp5DnwJulvRvwK+BayJi4Obzl8h6Jk0CfpheAEuA1ZKuBF4BLmnCbplV5GRgVqKWoV4j4n7g/iHmbQZOKVP+GnD2SGI0azQ3E5mZmZOBmZk5GZiZGU4GZmaGk4GZmeFkYGZmOBmYmRlOBmZmhpOBmZnhZGBmZjgZmJkZTgZmZoaTgZmZ4WRgZmY4GZhZg3UufoTOxY/Qu+sNOssMpWmtycnAzMycDMzKkbRc0l5Jz+bKjpG0XtLW9H50Kpek2yX1SXpG0mm5Zeal+lslzcuVny6pNy1zexoa06wwTgZm5a0AZpeULQY2RMR0YEP6DHAe2djH04EFwB2QJQ+yITNnAWcANw4kkFTnqtxypdsyayonA7MyIuIxYF9J8RxgZZpeCVyUK787MhuBoyQdD5wLrI+IfRGxH1gPzE7zPhwRGyMigLtz6zIrhMdANqteR0TsTtOvAh1pejKwI1dvZyqrVL6zTPkgkhaQXW3Q0dFBT0/PyPaggkUz+geVlW6vtE65eAbqdEzKpkcz5tF04MCBcRX7sMlA0nLgQmBvRJySyo4B7gM6ge3AJRGxP7V73gacD7wDzI+Ip9Iy84CvpdV+PSJWpvLTyS7JJwFrgevS2ZJZy4qIkDTqx2lELAOWAXR1dUV3d/eobWt+mZ4/2y/rrlindH6+zqIZ/SztnVi2Tjvo6elhNL/v0VRP7NU0E63AbadmAHtSEw/pfW8q3wVMzdWbksoqlU8pU25WmGGTgdtOzd63BhjoETQPeChXfkXqVXQm8EZqTloHnCPp6HTycw6wLs17U9KZ6Wr6ity6zApR7z2DpredQnPbT6vRrDbFatpyG2WgnXe0t9NM9fw7SboX6AaOlbST7Mp2CbBa0pXAK8AlqfpasqbRPrLm0S8ARMQ+SbcAm1K9myNi4MTqSxxsHv1hepkVZsQ3kJvVdpq21bT202o0q02xmrbcRvnWPQ+xtPfgYdGu7b159fw7RcSlQ8w6u0zdAK4dYj3LgeVlyjcDp9QUlNkoqrdrqdtOzczGkHqTgdtOzczGkGq6lrrt1MxsjBs2Gbjt1Mxs7PPjKMzMzMnAzMycDMzMDCcDMzPDycDMzHAyMDMznAzMzAwnAzMzw8nAzMxwMjAzM5wMzMwMJwMzM8PJwKxqkk6UtCX3elPS9ZJukrQrV35+bpkbJPVJelHSubny2amsT9Li8ls0a54Rj3RmNl5ExIvATABJE8gGYnqQ7FHt34yIv83Xl3QSMBc4GTgB+GdJH0+zvw18hmyo102S1kTE883YD7NynAzM6nM28FJEvJKNy1TWHGBVRLwLvCypDzgjzeuLiG0Aklaluk4GVhgnA7P6zAXuzX1eKOkKYDOwKCL2A5OBjbk6O1MZwI6S8lnlNiJpAbAAoKOjg56enoYEX86iGf2Dykq3V1qnXDwDdTomZdOjGfNoOnDgwLiK3cnArEaSDgc+C9yQiu4AbgEivS8FvtiIbUXEMmAZQFdXV3R3dzditWXNX/zIoLLtl3VXrFM6P19n0Yx+lvZOLFunHfT09DCa3/doqid2JwOz2p0HPBURewAG3gEk3Qk8nD7uAqbmlpuSyqhQblYI9yYyq92l5JqIJB2fm3cx8GyaXgPMlXSEpGnAdOAJsrHAp0ualq4y5qa6ZoXxlYFZDSQdSdYL6Opc8V9LmknWTLR9YF5EPCdpNdmN4X7g2oh4L61nIbAOmAAsj4jnmrUPZuU4GZjVICLeBj5SUnZ5hfq3AreWKV8LrG14gGZ1cjORmZk5GZiZmZOBmZnhZGBmZjgZmJkZTgZmZoa7llpBOss9+mDJBQVEYmbgKwMzM8PJwMzMcDIwMzOcDMzMDCcDMzPDycDMzHAyMDMznAzMzAwnAzMzw8nArGaStkvqlbRF0uZUdoyk9ZK2pvejU7kk3S6pT9Izkk7LrWdeqr9V0ryi9scMnAzM6vXpiJgZEV3p82JgQ0RMBzakzwDnkY19PB1YANwBWfIAbgRmAWcANw4kELMijCgZ+AzJ7H1zgJVpeiVwUa787shsBI6SdDxwLrA+IvZFxH5gPTC7yTGbva8RD6r7dET8Mvd54AxpiaTF6fNXOPQMaRbZGdKs3BlSF9mA4k9KWpP+QMxaUQA/khTAdyNiGdAREbvT/FeBjjQ9GdiRW3ZnKhuq/BCSFpBdUdDR0UFPT08Dd+NQi2b0Dyor3V5pnXLxDNTpmJRNj2bMo+nAgQPjKvbReGrpHKA7Ta8EesiSwftnSMBGSQNnSN2kMyQASQNnSPeOQmxmjfAHEbFL0r8D1kv6eX5mRERKFCOWEs0ygK6uruju7m7EasuaX+5Jspd1V6xTOj9fZ9GMfpb2Tixbpx309PQwmt/3aKon9pEmg6adIUFzz5Kq0awzh2rO2Bpl4GxutLfTzH1q9L9TROxK73slPUjW5r9H0vERsTud5OxN1XcBU3OLT0lluzh40jRQ3rggzWo00mTQtDOktL6mnSVVo1lnDtWcsTXKt+55iKW9Bw+L0dpOM/epkf9Oko4EPhARb6Xpc4CbgTXAPGBJen8oLbIGWChpFVnz6BspYawD/nvupvE5wA0NCdKsDiNKBj5DsnGoA3hQEmR/P/8QEY9K2gSslnQl8ApwSaq/Fjgf6APeAb4AEBH7JN0CbEr1bh5oKjUrQt3JwGdINh5FxDbgk2XKXwPOLlMewLVDrGs5sLzRMZrVYyRXBj5DMjMbI+pOBj5DMjMbO/wLZDMzG5XfGYwbvbveOKRXzPYlFxQYjZlZ/XxlYGZmTgZmZuZkYGZmOBmYmRlOBmZmhpOBmZnhZGBmZjgZmJkZTgZmZoaTgZmZ4WRgZmY4GZiZGX5QnVnVJE0F7iYbyyOAZRFxm6SbgKuAX6SqX42ItWmZG4ArgfeAL0fEulQ+G7gNmADcFRFLmrkvRessGfbUD3ksnpOBWfX6gUUR8ZSkDwFPSlqf5n0zIv42X1nSScBc4GTgBOCfJX08zf428BlgJ7BJ0pqIeL4pe2FWhpOBWZUiYjewO02/JekFYHKFReYAqyLiXeBlSX1k44QD9KUBokhDwc4BnAysME4GZnWQ1AmcCjwOnEU2vvcVwGayq4f9ZIliY26xnRxMHjtKymcNsZ0FwAKAjo4Oenp6GrcTJRbN6B9UVrq90jrl4hmo0zEpm65Up9J6inbgwIGWjKsa9cTuZGBWI0kfBO4Hro+INyXdAdxCdh/hFmAp8MVGbCsilgHLALq6uqK7u7sRqy1rfkk7PsD2y7or1imdn6+zaEY/S3snVqxTaT1F6+npYTS/79FUT+xOBmY1kHQYWSK4JyIeAIiIPbn5dwIPp4+7gKm5xaekMiqUmxXCXUvNqiRJwPeAFyLiG7ny43PVLgaeTdNrgLmSjpA0DZgOPAFsAqZLmibpcLKbzGuasQ9mQ/GVgVn1zgIuB3olbUllXwUulTSTrJloO3A1QEQ8J2k12Y3hfuDaiHgPQNJCYB1Z19LlEfFc83bDbDAnA7MqRcRPAZWZtbbCMrcCt5YpX1tpObNmczIwGwdKf+QF/qGXHcr3DMzMzMnAzMycDMzMDCcDMzPDycDMzHAyMDMznAzMzAwnAzMzw8nAzMxwMjAzM5wMzMwMP5vIzFqQn6XUfL4yMDMzJwMzM3MyMDMzWigZSJot6UVJfZIWFx2P2WjzMW+tpCVuIEuaAHwb+AywE9gkaU1EPF9sZNZuyt14XDH7yAIiqazRx3zpfo/Xm63+HurXEskAOAPoi4htAJJWAXPIxo6tiXshWJto2DFv1giKiKJjQNLngdkR8Z/T58uBWRGxsKTeAmBB+ngi8GJTAx3sWOCXBcfQaONpnz4WEcc1Oxho62O+Fu1+LLVz/JViL3vct8qVQVUiYhmwrOg4BkjaHBFdRcfRSN6n1tJqx3wt2vl7h/aOv57YW+UG8i5gau7zlFRmNlb5mLeW0irJYBMwXdI0SYcDc4E1BcdkNpp8zFtLaYlmoojol7QQWAdMAJZHxHMFh1WNtrx8H4b3qQna+JivRct97zVq5/hrjr0lbiCbmVmxWqWZyMzMCuRkYGZmTga1kjRV0o8lPS/pOUnXFR1To0iaIOlpSQ8XHUsjSDpK0vcl/VzSC5L+Q9ExjReStkvqlbRF0uai46lE0nJJeyU9mys7RtJ6SVvT+9FFxljJEPHfJGlX+v63SDp/uPU4GdSuH1gUEScBZwLXSjqp4Jga5TrghaKDaKDbgEcj4t8Dn2Rs7Vs7+HREzGyDvvorgNklZYuBDRExHdiQPreqFQyOH+Cb6fufGRFrh1uJk0GNImJ3RDyVpt8i+w9mcrFRjZykKcAFwF1Fx9IIkn4L+BTwPYCI+FVEvF5oUNaSIuIxYF9J8RxgZZpeCVzUzJhqMUT8NXMyGAFJncCpwOMFh9IIfwf8JfDrguNolGnAL4C/T01fd0lqvSfWjV0B/EjSk+mRGu2mIyJ2p+lXgY4ig6nTQknPpGakYZu5nAzqJOmDwP3A9RHxZtHxjISkC4G9EfFk0bE00ETgNOCOiDgVeJvWvtQfa/4gIk4DziNrSv1U0QHVK7L+9+3WB/8O4HeBmcBuYOlwCzgZ1EHSYWSJ4J6IeKDoeBrgLOCzkrYDq4A/kvS/iw1pxHYCOyNi4Krt+2TJwZogInal973Ag2RPaW0neyQdD5De9xYcT00iYk9EvBcRvwbupIrv38mgRpJE1g79QkR8o+h4GiEiboiIKRHRSfZYhH+JiD8vOKwRiYhXgR2STkxFZ+PHQzeFpCMlfWhgGjgHeLbyUi1nDTAvTc8DHiowlpoNJLLkYqr4/lvicRRt5izgcqBX0pZU9tVq7tZb0/0FcE969s824AsFxzNedAAPZudNTAT+ISIeLTakoUm6F+gGjpW0E7gRWAKslnQl8ApwSXERVjZE/N2SZpI1b20Hrh52PX4chZmZuZnIzMycDMzMzMnAzMxwMjAzM5wMzMwMJwMzM8PJwMzMgP8PLq3eGzymXqAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "# empty lists\n",
        "eng_l = []\n",
        "deu_l = []\n",
        "\n",
        "# populate the lists with sentence lengths\n",
        "for i in deu_eng[:,0]:\n",
        "      eng_l.append(len(i.split()))\n",
        "\n",
        "for i in deu_eng[:,1]:\n",
        "      deu_l.append(len(i.split()))\n",
        "\n",
        "length_df = pd.DataFrame({'eng':eng_l, 'deu':deu_l})\n",
        "\n",
        "length_df.hist(bins = 30)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 2.3 Tokenize and encode"
      ],
      "metadata": {
        "id": "2hH_wE5uAcxM"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tc51f_p2ehJV"
      },
      "source": [
        "Next, vectorize our text data by using Keras’s Tokenizer() class. It will turn our sentences into sequences of integers. We can then pad those sequences with zeros to make all the sequences of the same length.\n",
        "\n",
        "Note that we will prepare tokenizers for both the German and English sentences:\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "G34GTpuDdX9c"
      },
      "outputs": [],
      "source": [
        "# function to build a tokenizer\n",
        "def tokenization(lines):\n",
        "      tokenizer = Tokenizer()\n",
        "      tokenizer.fit_on_texts(lines)\n",
        "      return tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "60Tdj8ZPela9",
        "outputId": "483134bf-15bc-4184-dd74-8584889fed39"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "English Vocabulary Size: 6106\n"
          ]
        }
      ],
      "source": [
        "# prepare english tokenizer\n",
        "eng_tokenizer = tokenization(deu_eng[:, 0])\n",
        "eng_vocab_size = len(eng_tokenizer.word_index) + 1\n",
        "\n",
        "eng_length = 8\n",
        "print('English Vocabulary Size: %d' % eng_vocab_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "imL2C1sGenaF",
        "outputId": "90f88311-19d7-4c0a-9184-5bd636cd9a0a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Deutch Vocabulary Size: 10060\n"
          ]
        }
      ],
      "source": [
        "# prepare Deutch tokenizer\n",
        "deu_tokenizer = tokenization(deu_eng[:, 1])\n",
        "deu_vocab_size = len(deu_tokenizer.word_index) + 1\n",
        "\n",
        "deu_length = 8\n",
        "print('Deutch Vocabulary Size: %d' % deu_vocab_size)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U0VFxjnvfILC"
      },
      "source": [
        "The below code block contains a function to prepare the sequences. It will also perform sequence padding to a maximum sentence length as mentioned above."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gp1Q89oHfFhy"
      },
      "outputs": [],
      "source": [
        "# encode and pad sequences\n",
        "def encode_sequences(tokenizer, length, lines):\n",
        "         # integer encode sequences\n",
        "         seq = tokenizer.texts_to_sequences(lines)\n",
        "         # pad sequences with 0 values\n",
        "         seq = pad_sequences(seq, maxlen=length, padding='post')\n",
        "         return seq"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X0gzTJk3fMmL"
      },
      "source": [
        "## 2.4 Training Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4QB00yTVfMOm"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# split data into train and test set\n",
        "train, test = train_test_split(deu_eng, test_size=0.2, random_state = 12)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nIJPhxeUfhFN"
      },
      "source": [
        "It’s time to encode the sentences. We will encode German sentences as the input sequences and English sentences as the target sequences. This has to be done for both the train and test datasets."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BiDECWb3fim0"
      },
      "outputs": [],
      "source": [
        "# prepare training data\n",
        "trainX = encode_sequences(deu_tokenizer, deu_length, train[:, 1])\n",
        "trainY = encode_sequences(eng_tokenizer, eng_length, train[:, 0])\n",
        "\n",
        "# prepare validation data\n",
        "testX = encode_sequences(deu_tokenizer, deu_length, test[:, 1])\n",
        "testY = encode_sequences(eng_tokenizer, eng_length, test[:, 0])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CTICyh4HfxHv"
      },
      "source": [
        "We’ll start off by defining our Seq2Seq model architecture:\n",
        "\n",
        "\n",
        "\n",
        "*   For the encoder, we will use an embedding layer and an LSTM layer.\n",
        "*   For the decoder, we will use another LSTM layer followed by a dense layer.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BBOqbtipfwxt"
      },
      "outputs": [],
      "source": [
        "# build NMT model\n",
        "def define_model(in_vocab,out_vocab, in_timesteps,out_timesteps,units):\n",
        "      model = Sequential()\n",
        "      model.add(Embedding(in_vocab, units, input_length=in_timesteps, mask_zero=True))\n",
        "      model.add(LSTM(units))\n",
        "      model.add(RepeatVector(out_timesteps))\n",
        "      model.add(LSTM(units, return_sequences=True))\n",
        "      model.add(Dense(out_vocab, activation='softmax'))\n",
        "      return model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H_qAKlhpf9Th"
      },
      "source": [
        "We are using the RMSprop optimizer in this model as it’s usually a good choice when working with recurrent neural networks."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Zlu42Zwnf7F5",
        "outputId": "0ee42edc-c173-45c9-ba8f-5351e2b9ae50"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2022-08-14 13:56:27.630988: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
            "2022-08-14 13:56:27.631902: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Metal device set to: Apple M1\n"
          ]
        }
      ],
      "source": [
        "# model compilation\n",
        "model = define_model(deu_vocab_size, eng_vocab_size, deu_length, eng_length, 512)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "B-VuuuAiga3Q"
      },
      "outputs": [],
      "source": [
        "from tensorflow.keras import optimizers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5JGwVdregB-a"
      },
      "outputs": [],
      "source": [
        "rms = optimizers.RMSprop(learning_rate=0.001)\n",
        "model.compile(optimizer=rms, loss='sparse_categorical_crossentropy')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u5Dhhy-FyCuQ",
        "outputId": "bf43e27a-244f-44f7-a524-e0d8a79ff6b5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding (Embedding)       (None, 8, 512)            5150720   \n",
            "                                                                 \n",
            " lstm (LSTM)                 (None, 512)               2099200   \n",
            "                                                                 \n",
            " repeat_vector (RepeatVector  (None, 8, 512)           0         \n",
            " )                                                               \n",
            "                                                                 \n",
            " lstm_1 (LSTM)               (None, 8, 512)            2099200   \n",
            "                                                                 \n",
            " dense (Dense)               (None, 8, 6106)           3132378   \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 12,481,498\n",
            "Trainable params: 12,481,498\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qcQkkXAygn8b",
        "outputId": "12853d21-cb52-4693-86e6-0a117c131758"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/30\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2022-08-14 13:56:33.025856: W tensorflow/core/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n",
            "2022-08-14 13:56:35.003165: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n",
            "2022-08-14 13:56:35.510898: W tensorflow/core/common_runtime/forward_type_inference.cc:231] Type inference failed. This indicates an invalid graph that escaped type checking. Error message: INVALID_ARGUMENT: expected compatible input types, but input 1:\n",
            "type_id: TFT_OPTIONAL\n",
            "args {\n",
            "  type_id: TFT_PRODUCT\n",
            "  args {\n",
            "    type_id: TFT_TENSOR\n",
            "    args {\n",
            "      type_id: TFT_INT32\n",
            "    }\n",
            "  }\n",
            "}\n",
            " is neither a subtype nor a supertype of the combined inputs preceding it:\n",
            "type_id: TFT_OPTIONAL\n",
            "args {\n",
            "  type_id: TFT_PRODUCT\n",
            "  args {\n",
            "    type_id: TFT_TENSOR\n",
            "    args {\n",
            "      type_id: TFT_FLOAT\n",
            "    }\n",
            "  }\n",
            "}\n",
            "\n",
            "\twhile inferring type of node 'cond_40/output/_23'\n",
            "2022-08-14 13:56:35.514311: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n",
            "2022-08-14 13:56:36.840359: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n",
            "2022-08-14 13:56:38.657833: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n",
            "2022-08-14 13:56:39.970511: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "63/63 [==============================] - ETA: 0s - loss: 3.2076"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2022-08-14 13:56:54.092429: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n",
            "2022-08-14 13:56:54.271918: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n",
            "2022-08-14 13:56:54.342480: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 23s 247ms/step - loss: 3.2076 - val_loss: 2.7596\n",
            "Epoch 2/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 2.7141WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 14s 226ms/step - loss: 2.7141 - val_loss: 2.6740\n",
            "Epoch 3/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 2.5493WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 14s 229ms/step - loss: 2.5493 - val_loss: 2.5420\n",
            "Epoch 4/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 2.3936WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 15s 233ms/step - loss: 2.3936 - val_loss: 2.3853\n",
            "Epoch 5/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 2.2692WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 14s 226ms/step - loss: 2.2692 - val_loss: 2.3101\n",
            "Epoch 6/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 2.1607WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 14s 225ms/step - loss: 2.1607 - val_loss: 2.2186\n",
            "Epoch 7/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 2.0523WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 14s 221ms/step - loss: 2.0523 - val_loss: 2.1389\n",
            "Epoch 8/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 1.9512WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 14s 224ms/step - loss: 1.9512 - val_loss: 2.0498\n",
            "Epoch 9/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 1.8570WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 14s 227ms/step - loss: 1.8570 - val_loss: 2.0166\n",
            "Epoch 10/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 1.7714WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 14s 227ms/step - loss: 1.7714 - val_loss: 1.9507\n",
            "Epoch 11/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 1.6927WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 14s 224ms/step - loss: 1.6927 - val_loss: 1.9166\n",
            "Epoch 12/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 1.6192WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 14s 225ms/step - loss: 1.6192 - val_loss: 1.8566\n",
            "Epoch 13/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 1.5476WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 15s 236ms/step - loss: 1.5476 - val_loss: 1.8266\n",
            "Epoch 14/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 1.4787WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 14s 222ms/step - loss: 1.4787 - val_loss: 1.7738\n",
            "Epoch 15/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 1.4099WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 16s 248ms/step - loss: 1.4099 - val_loss: 1.7451\n",
            "Epoch 16/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 1.3411WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 15s 241ms/step - loss: 1.3411 - val_loss: 1.6968\n",
            "Epoch 17/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 1.2776WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 16s 254ms/step - loss: 1.2776 - val_loss: 1.6560\n",
            "Epoch 18/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 1.2117WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 15s 245ms/step - loss: 1.2117 - val_loss: 1.6437\n",
            "Epoch 19/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 1.1484WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 15s 244ms/step - loss: 1.1484 - val_loss: 1.5966\n",
            "Epoch 20/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 1.0895WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 15s 243ms/step - loss: 1.0895 - val_loss: 1.5829\n",
            "Epoch 21/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 1.0270WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 16s 257ms/step - loss: 1.0270 - val_loss: 1.5383\n",
            "Epoch 22/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 0.9697WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 16s 249ms/step - loss: 0.9697 - val_loss: 1.5235\n",
            "Epoch 23/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 0.9137WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 16s 255ms/step - loss: 0.9137 - val_loss: 1.5028\n",
            "Epoch 24/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 0.8576WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 16s 253ms/step - loss: 0.8576 - val_loss: 1.4748\n",
            "Epoch 25/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 0.8044WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 15s 245ms/step - loss: 0.8044 - val_loss: 1.4775\n",
            "Epoch 26/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 0.7525WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 16s 249ms/step - loss: 0.7525 - val_loss: 1.4426\n",
            "Epoch 27/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 0.7047WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 15s 246ms/step - loss: 0.7047 - val_loss: 1.4197\n",
            "Epoch 28/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 0.6579WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 16s 255ms/step - loss: 0.6579 - val_loss: 1.4176\n",
            "Epoch 29/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 0.6116WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 16s 248ms/step - loss: 0.6116 - val_loss: 1.3976\n",
            "Epoch 30/30\n",
            "63/63 [==============================] - ETA: 0s - loss: 0.5680WARNING:tensorflow:Can save best model only with val_loss, val_accuracy available, skipping.\n",
            "63/63 [==============================] - 15s 244ms/step - loss: 0.5680 - val_loss: 1.3942\n"
          ]
        }
      ],
      "source": [
        "filename = 'model.h1.24_jan_19'\n",
        "checkpoint = ModelCheckpoint(filename, monitor='val_loss, val_accuracy', verbose=1, save_best_only=True, mode='min')\n",
        "\n",
        "# train model\n",
        "history = model.fit(trainX, trainY.reshape(trainY.shape[0], trainY.shape[1], 1),\n",
        "                    epochs=30, batch_size=512, validation_split = 0.2,callbacks=[checkpoint], \n",
        "                    verbose=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z7llvGncrkUw",
        "outputId": "734763df-3d33-446b-f24a-d65b6df64eab"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD4CAYAAAATpHZ6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAulklEQVR4nO3dd3xUVf7/8ddJh4SENAhpJAECISEQCAGpQQQRFRGlWNYOdnTd9fv162/3q1u/bnMVu9gbiKAoCItKRxRIqAECoSSQHggkpJc5vz/uAAHTmWQyk8/z8ZjHzNx7587nOvrO8dxzz1Vaa4QQQtgHB2sXIIQQwnIk1IUQwo5IqAshhB2RUBdCCDsioS6EEHbEyVpf7Ofnp8PCwqz19UIIYZOSk5NPaa39G1pvtVAPCwsjKSnJWl8vhBA2SSmV0dh66X4RQgg7IqEuhBB2REJdCCHsiNX61IUQ9qW6uprMzEwqKiqsXYpdcHNzIzg4GGdn5xZ9TkJdCGERmZmZdOvWjbCwMJRS1i7HpmmtOX36NJmZmYSHh7fos9L9IoSwiIqKCnx9fSXQLUApha+vb6v+r0dCXQhhMRLoltPaf5Y2F+qH887xp5UHqKiutXYpQgjR4dhcqGeeKePdLcfZkV5o7VKEEB3I2bNnef3111v8ualTp3L27FnLF2QlNhfqV0X44eLkwIZDBdYuRQjRgTQU6jU1NY1+btWqVXTv3r2Nqmp/NhfqXVwcGRHuw8bDEupCiIueeeYZjh49ypAhQxg+fDhjx45l2rRpDBw4EIDp06czbNgwoqOjefvtty98LiwsjFOnTpGenk5UVBRz584lOjqayZMnU15ebq3DaTWbHNI4PtKfP397kMwzZQR7d7V2OUKIy/xhxX4OZBdbdJ8DAz157sboBte/8MILpKSksHv3bjZs2MD1119PSkrKhSGB7733Hj4+PpSXlzN8+HBuueUWfH19L9lHWloaixYtYuHChcyaNYtly5Zx5513WvQ42prNtdQBEvv3AJAuGCFEgxISEi4Z471gwQIGDx7MyJEjOXnyJGlpab/4THh4OEOGDAFg2LBhpKent1O1lmOTLfU+/u4Ee3dh4+EC7hzZ29rlCCEu01iLur24u7tfeL1hwwZ++OEHfvrpJ7p27UpiYmK9Y8BdXV0vvHZ0dLTJ7hebbKkrpUjs78/WI6eoqjFZuxwhRAfQrVs3zp07V++6oqIivL296dq1K6mpqfz888/tXF37sclQBxgf2YPSqlqSZGijEALw9fVl9OjRxMTE8PTTT1+ybsqUKdTU1BAVFcUzzzzDyJEjrVRl27PJ7heAUX18cXF0YMPhAkb19bN2OUKIDuCzzz6rd7mrqyurV6+ud935fnM/Pz9SUlIuLP/tb39r8frag8221N1dnRge7s1GOVkqhBAX2GyogzG08VDeObLP2t7JDCGEaAs2HernhzbKhUhCCGGw6VDv18ODQC836YIRQgizJkNdKeWmlNqulNqjlNqvlPpDPdu4KqU+V0odUUptU0qFtUm1v/xexvf358cjp6iulaGNQgjRnJZ6JXC11nowMASYopS6fDzQ/cAZrXVf4N/A3yxaZSPGR/bgXGUNyRln2usrhRCiw2oy1LWhxPzW2fzQl212E/Ch+fVSYKJqp9nyR/f1xclByZQBQogW8fDwACA7O5tbb7213m0SExNJSkpqdD8vvfQSZWVlF95beyrfZvWpK6UclVK7gXzge631tss2CQJOAmita4AiwPeybVBKzVNKJSmlkgoKLBPC3dyciQ/zlpOlQohWCQwMZOnSpa3+/OWhbu2pfJsV6lrrWq31ECAYSFBKxbTmy7TWb2ut47XW8f7+/q3ZRb3GR/bgYE4xecVyF3MhOqtnnnmG11577cL7559/nj//+c9MnDiRoUOHMmjQIL7++utffC49PZ2YGCPSysvLmTNnDlFRUdx8882XzP3y8MMPEx8fT3R0NM899xxgTBKWnZ3NhAkTmDBhAnBxKl+AF198kZiYGGJiYnjppZcufF9bTvHboitKtdZnlVLrgSlASp1VWUAIkKmUcgK8gNMWq7IJif39+dt/Utl4qIBZw0Pa62uFEA1Z/Qzk7rPsPgMGwXUvNLh69uzZPPnkkzz66KMALFmyhDVr1jB//nw8PT05deoUI0eOZNq0aQ3e//ONN96ga9euHDx4kL179zJ06NAL6/7yl7/g4+NDbW0tEydOZO/evcyfP58XX3yR9evX4+d36ZXtycnJvP/++2zbtg2tNSNGjGD8+PF4e3u36RS/zRn94q+U6m5+3QWYBKRettk3wN3m17cC67TWl/e7t5kBAd3o6ekqXTBCdGJxcXHk5+eTnZ3Nnj178Pb2JiAggGeffZbY2FiuueYasrKyyMvLa3AfmzZtuhCusbGxxMbGXli3ZMkShg4dSlxcHPv37+fAgQON1rNlyxZuvvlm3N3d8fDwYMaMGWzevBlo2yl+m9NS7wV8qJRyxPgjsERrvVIp9UcgSWv9DfAu8LFS6ghQCMyxWIXNoJRifKQ//0nJpabWhJOjTQ+/F8L2NdKibkszZ85k6dKl5ObmMnv2bD799FMKCgpITk7G2dmZsLCweqfcbcrx48f55z//yY4dO/D29uaee+5p1X7Oa8spfpsz+mWv1jpOax2rtY7RWv/RvPx/zYGO1rpCaz1Ta91Xa52gtT5msQqbKbF/D4orath18mx7f7UQooOYPXs2ixcvZunSpcycOZOioiJ69OiBs7Mz69evJyMjo9HPjxs37sKkYCkpKezduxeA4uJi3N3d8fLyIi8v75LJwRqa8nfs2LEsX76csrIySktL+eqrrxg7dqwFj7Z+NjtL4+VG9/XD0UGx4VA+w8N8rF2OEMIKoqOjOXfuHEFBQfTq1Ys77riDG2+8kUGDBhEfH8+AAQMa/fzDDz/MvffeS1RUFFFRUQwbNgyAwYMHExcXx4ABAwgJCWH06NEXPjNv3jymTJlCYGAg69evv7B86NCh3HPPPSQkJADwwAMPEBcX1+Z3U1Lt2PV9ifj4eN3U+M+WmvnmVsqra1n5eNv/NRRCXOrgwYNERUVZuwy7Ut8/U6VUstY6vqHP2FXnc2L/HqRkFZN/ToY2CiE6J7sK9fGRxtj3TYdPWbkSIYSwDrsK9ehAT/y7ydBGIazFWt259qi1/yztKtSVUozr58/mtAJqTfIvlxDtyc3NjdOnT0uwW4DWmtOnT+Pm5tbiz9rN6JfzEvv7s2xnJrtPnmVYb29rlyNEpxEcHExmZiaWmteps3NzcyM4OLjFn7O7UB/bzw8HZdwNSUJdiPbj7OxMeHi4tcvo9Oyq+wWge1cXhoR0Z+OhfGuXIoQQ7c7uQh2MoY17s4o4XVJp7VKEEKJd2Waolzd+l6PE/v5oDZvSpG9PCNG52F6oH/4OXoqFbW9BbU29m8QEeuHr7iI3pBZCdDq2F+q+fSA4Hlb/FyycAJm/nGrAwUExLtKfTWmnMMnQRiFEJ2KboX7nlzDzAygtgHeugRVPQlnhJZsl9vensLSKvVlFVilTCCGswfZCHUApiL4ZHt0OIx+BnR/Bq8Nh92dgvvBhbD9/lIJ1qTIKRgjRedhmqJ/n5glT/goPbgSfCFj+MLw/FfIO4OPuwpi+fizcdIx9mdJaF0J0DrYd6ucFDIL71sC0V6DgILw1Fr77PS9O74uPuwv3f7iDnCLL3VlECCE6KvsIdQAHBxh6FzyWDENuh60L8P9wLIsnllFWVcv9HyRRWln/aBkhhLAX9hPq57n7Gi32+78HNy9CVt/NklGZpOYW88TiXTLRlxDCrtlfqJ8XkmB0yYSMYOBPv+GzuIP8cDCfv646aO3KhBCizdhvqINxIvXOZdBvEiMP/Im3+2zl3S3H+fjnxm8+K4QQtsq+Qx3AuQvM/hSib2Zy1qss6LmK579JkRtpCCHskt1NvVsvJxe45V1w8WDaro8xeRTx+KcOLH1kDJE9u1m7OiGEsJjOEeoADo7GCVRXT6b//BqODiU88J4Tyx4bh383V2tXJ4QQFtF5Qh2MK1Gv/Qu4eXLjhv/DuaKMRz5y5uN5Y3FzdrR2dUIIccXsv0/9ckpB4jNw7V+ZorbxaO7/8j+fb5OJv4QQdqHzhfp5Vz0K015hvONe5hz+Na//Z6e1KxJCiCvWeUMdjCtQb32PeIc0xv98H8t+TLF2RUIIcUWaDHWlVIhSar1S6oBSar9S6ol6tklUShUppXabH//bNuVanoqZgZ7zGVEOmQSvuZ91+05YuyQhhGi15rTUa4DfaK0HAiOBR5VSA+vZbrPWeoj58UeLVtnGnAZMoeamNxjhkErV0gdIPn7K2iUJIUSrNBnqWuscrfVO8+tzwEEgqK0La29ucbMoSfwDU9Q2Dn34GEfyiq1dkhBCtFiL+tSVUmFAHLCtntVXKaX2KKVWK6WiG/j8PKVUklIqqaCg413R6ZH4JMVDHuR2VvPdwmfJLaqwdklCCNEizQ51pZQHsAx4Umt9eTN2J9Bbaz0YeAVYXt8+tNZva63jtdbx/v7+rSy5bXlOe4GzfW7ikZqP+ejNFygqr7Z2SUII0WzNCnWllDNGoH+qtf7y8vVa62KtdYn59SrAWSnlZ9FK24uDA91vW8jZnlfx67IFvLrwTSqqa61dlRBCNEtzRr8o4F3goNb6xQa2CTBvh1Iqwbzf05YstF05udL93iWUde/Hk6f/xIsfLpF52IUQNqE5LfXRwK+Aq+sMWZyqlHpIKfWQeZtbgRSl1B5gATBHa23bKejmidcDX2Pq4sPck//Fy1+swdYPSQhh/5S1gio+Pl4nJSVZ5btb5FQa5W9cTW51F9aN+pj7p4ywdkVCiE5MKZWstY5vaH3nvqK0Ofz64XrXUoIczzJs60Ms+ynV2hUJIUSDJNSbwaH3CBxmvkesQzq+q+bx3b6T1i5JCCHqJaHeTE4Db6D6un+R6LgH9y/msGbt99YuSQghfkFCvQVcR9xHxZR/Mdgpg0mbZnLwtdnowuPWLksIIS6QUG8ht5EP4Pqbfaz3v52w/HXUvhJP7ar/glKZL0YIYX0S6q3g7O7N1Y++xsfxX7KkeixsX4h+ORY2/A0qS6xdnhCiE5NQbyWlFPNuHAs3LuDaqr/xE4Nhw19hwRDY9jbUVFm7RCFEJyShfoVuHxHKM3fexH3l83nQ9e+Ue/WF1U/Da8Nh31KQC5aEEO1IQt0CrhnYk0VzR7KjJoLRuU9xdNL74OIBy+6Hj2+GMxnWLlEI0UlIqFtIXKg3yx4eRbcuzlz/ny78MG4pTP0nZO6A168yumRMJmuXKYSwcxLqFhTu586yh0cR2bMb8z7Zyad6MjzyE4SOMLpkPpgKp45Yu0whhB2TULcwPw9XFs0dyfhIf/7fVyn8fkMxlXO+gJteh/wD8MYo2PIS1NZYu1QhhB2SUG8D7q5OLLwrnnnjIvj45wxmvb2NrPAZ8Oh26DcJfngO3pkIuSnWLlUIYWck1NuIk6MDz06N4s07h3I0v4QbFmxmU44jzP4EZn4AxVnw9nhY/1cZ/iiEsBgJ9TY2JaYX3zw2mh7d3Lj7/e28su4IpqjpRqs95hbY+Dd4axyc3G7tUoUQdkBCvR1E+Hvw1aOjmD4kiH99f5j7P9zBWTxgxttw+xKoLIZ3J8HS++GszAAphGg9CfV20tXFiRdnDeZP02PYcuQUN7yyhX2ZRRB5rdFqH/c0pK6EV+Nh3V9kugEhRKtIqLcjpRS/GtmbLx4ahcmkueXNrSzefgJcPeDq38FjSTDgBtj0d3hlGOz+TMa2CyFaRELdCoaEdGfl/LGMCPfhmS/38fQXe6ioroXuIXDru3Dfd+AVBMsfhoUTIOMna5cshLAREupW4uPuwgf3JjB/Yj++SM5kxutbyThdaqwMHQH3/wAzFkJJPrw/BZbcDWfSrVqzEKLjk1C3IkcHxVOTInn/nuFknS3nhle28MOBPGOlgwPEzoLHkyDxfyDtO3g1AX54HqpKrVq3EKLjklDvACYM6MHKx8cQ5uvOAx8l8Y81qdSazLM7urhD4jNGf3v0zbDl38ZcMkfWWrdoIUSHJKHeQYT4dOWLh67itoQQXlt/lLve28apksqLG3gFwYy34J5V4OgCn8yAL+fJHZeEEJeQUO9A3Jwd+b8Zsfz91liS0s9ww4ItJGecuXSjsNHw0BYY/9+Q8iW8Ohx2L5J524UQgIR6hzQrPoQvHxmFi5MDs9/6iQ9+PI6uG9rObjDhWXhoM/j2heUPGfO2y02whej0JNQ7qOhAL1Y8NobE/v48v+IA8xfvprTyspkde0TBfWvM87YnGX3tMgOkEJ2ahHoH5tXVmbd/Fc/T1/bn273Z3PTajxzJv+xKUwcHSJgLj26DPlcbM0AuTITsXVapWQhhXUpbqS82Pj5eJyUlWeW7bdGPR04xf9EuKqpr+dP0GGYMDa5/wwPfwKqnoTQfIq+D3qOg91UQEAuOzu1btBDC4pRSyVrr+AbXS6jbjpyicp5YtJvt6YVMHxLIn6bH0M2tnqAuP2vM/nho1cULlpzdITgeQq8yQj54uDFcUghhU6441JVSIcBHQE9AA29rrV++bBsFvAxMBcqAe7TWOxvbr4R669SaNK+tP8LLa9MI7O7Gy3PiGBrq3fAHirPhxM9w4ifjkZsCaFCO0Guw0ZIPGwN9J4GjU7sdhxCidSwR6r2AXlrrnUqpbkAyMF1rfaDONlOBxzFCfQTwstZ6RGP7lVC/MskZhcxftJvc4gqemhTJQ+P74Oigmv5gRRGc3AEnthphn5kEtZXgHQajn4DBtxuja4QQHZLFu1+UUl8Dr2qtv6+z7C1gg9Z6kfn9ISBRa53T0H4k1K9cUXk1v1uewoo92YyM8OHfs4fQy6tLy3ZSU2lMQbDl35CVDB49YeQjEH8fuHm2TeFCiFZrKtRbNPpFKRUGxAHbLlsVBNS9u0Omednln5+nlEpSSiUVFBS05KtFPby6OLNgzhD+cWssezOLuO7lzazZn9uynTi5QtSN8MBauHsF9BhojKD5dwys/SOUyO8khC1pdqgrpTyAZcCTWuvi1nyZ1vptrXW81jre39+/NbsQl1FKMTM+hG/njyXEuysPfpzMs1/to7yqtqU7gvBxcNdymLcB+iTC5hfhpRj49rdwJqMNqhdCWFqzQl0p5YwR6J9qrb+sZ5MsIKTO+2DzMtFOwv3cWfbwKB4cF8Fn204w7dUtHMhu1d9eCIyDWR/BYztg0ExI/gAWxBlzzRzfBNXlFq1dCGE5zTlRqoAPgUKt9ZMNbHM98BgXT5Qu0FonNLZf6VNvO1vSTvHrJbs5W1bF/Kv78VBiH5wdr+A6s+Js+Ok1SHofqkvBwRmChhnz0PQeBSEjwLWb5Q5ACNEgS4x+GQNsBvYB5++t9iwQCqC1ftMc/K8CUzCGNN6rtW40sSXU29aZ0iqe+2Y/3+zJZlCQF/+aNZjInlcYvBXFxrDIjB8h/UfjqlVdax4eGQu9RxuP0JHQ1ccyByKEuIRcfNTJrd6Xw++Wp3CuooZfT4pk7thwnK6k1V5XZQlk7jBCPmPrxeGRKAgYBMPuhtg5xj1YhRAWIaEuOF1Sye+/TmHVvlyGhHTnnzMH07dHGwRtdYUxLDJjK6SugJw94OoFcXdCwgPgE2H57xSik5FQFxes2JPN779Ooayqlqcn9+e+MeHNu2CpNbQ2WvHb3oQDX4OpFiKvhREPQsQEY7SNEKLFJNTFJfLPVfD/vkrh+wN5DOvtzT9ujSXCv427R4pzIOk941F2CvwiIWEeDL5NumaEaCEJdfELWmu+3p3Nc9/sp7KmlqevHcA9o8LartV+Xk0l7P/KaL1n7wJXT6NrJnqGcaLVybVtv18IOyChLhqUV1zBs1/uY21qPoOCvPjz9BgGh3Rv+y/W2jipuu1NOLAcTDXGMMmAQcZMkkHxxpBJ3z7STSPEZSTURaO01qzcm8OfVh6goKSSO0aE8vTkAXh1bae510sKjGGSWUmQmWy04KtLjXVu3Y1wDzaHfPBwGSopOj0JddEs5yqq+ff3aXyw9TjeXV14dmoUM4YGodq7pWyqhYJUoyV/PugLDoI2gXKAPhNhyO3Qf6rMJik6JQl10SL7s4v43fIUdp04S0K4D3+eHnPlFy1dqcoSyNkNR9bC3s+hOMtoxQ+61Qj4wKHSTSM6DQl10WImk2ZJ0kle+E8qJRU13D82nCcm9qOrSwe4iYapFo5vhF2fQupKqKkA/ygj3GNnQ7ee1q5QiDYloS5arbC0ihdWH2RJUiaBXm48Ny2ayQN7tn+XTEPKzxqjaXZ/BpnbjekK+l4DcXcYd3Jy6WrtCoWwOAl1ccWS0gv53fIUUnPPMS7Sn99dH2X9LpnLFRyGPZ/BnsVwLscI+B5REDjEmHUycCj0jJZhk8LmSagLi6iuNfHRTxm8/MNhSiprmJMQyq+vicS/WwcLSVMtHNtg3Koveydk7YTyQmOdgzMExJhD3hz0/gPk3qzCpkioC4s6U1rFy2vT+OTnDNycHXlkQh/uGx2Om7OjtUurn9Zw9oQxVDJ7p/l5N1Sa55p36QYDpkLMLcb0BU4uVi1XiKZIqIs2cayghP9bncr3B/II6t6F/5rSn2mDAztOf3tjTCYoPGaE/PGNcHCFcUPuLt4QNc0I+LAx4NBB/1CJTk1CXbSprUdP8eeVBzmQU8zgkO78/voo4sNs7AKhmio4ug5SlkHqt8bFTx49IfpmI+CDh8uQSdFhSKiLNldr0ny5M5N/rDlE/rlKrh/Ui/+eMoBQXxscfVJVBmlrjIA//J0xP7xXKMTMMFrxgXHgYKH56IVoBQl10W7Kqmp4a+Mx3t50jBqTiTtG9ObRCX073snU5qooNlruKcvg2HpjjpquftBvkvHoMxG6dLd2laKTkVAX7S63qIKXfjjMF8mZuDg6cN+YMOaN7dN+88m0hbJC44rWtDVw5AcoP2MMmwwZAZGTod9k6DFQumlEm5NQF1ZzrKCEf/+Qxoo92Xi6OfHg+D7cOzqsY1yZeiVMtcbcNGnfGSGfu89Y7hlstOAjEsEzCDz8wd0fXNytWq6wLxLqwur2Zxfxr+8Osy41Hz8PVx6/ui+3JYTi4mQnfdPF2ZD2vRHyxzZAVcml653dwd0PPHqAe486r/3N0wzL3DWi+STURYeRlF7I39ccYvvxQoK9u/DkNZHcHBfU9jfnaE81lZB/wJhSuDQfSgvMr8+/PwUl+cYdoLTJ+IxXKERPN0bbBMZJwItGSaiLDkVrzea0U/xjzSH2ZRXRt4cHT02KZEp0AA72FO5NMZmMoD+61pi/5ug640Ssd5gR7tE3Q0CsBLz4BQl10SFprVmzP5d/fneYI/klRPXy5KlJkVwT1cM2LmCytLJCY6TN/q+MLhxdCz4RFwO+Z4wEvAAk1EUHV2vSfLMni5d/SCP9dBmDg7349aRIxkf6d85wByg9bUwrvP8rOL7JCHivEOgealwU1S3g0ufzr7t4S/B3AhLqwibU1Jr4cmcWC9alkXmmnGG9vXlqUiSj+vh23nAHow/+4Aoj3M/lQkkunMu7eMu/uhxdjIAPGgZRNxrDLN08279m0aYk1IVNqaox8UXySV5dd4ScogpGhPvwm8n9SQi3sakH2lplCZTkXRr0JbnGSJxjG42Tso4uED4eom6A/tcbQyyFzZNQFzaporqWxdtP8NqGoxScq2RsPz9+PSmSoaHe1i6t4zPVQuYOo4WfuhLOpBv3dw0ZabTgo24wunKETZJQFzatvKqWT7dl8MaGo5wurWJcpD/zr+5re5OGWYvWkJcCB1caIZ+/31geEAuR10L33uDZC7oFGs9u3aVfvoO74lBXSr0H3ADka61j6lmfCHwNHDcv+lJr/cemCpNQFy1RWlnDxz9nsHDTMU6XVjGqjy/zJ/ZjZISvtUuzLaePGq33gyuN1jyX/ffv1MUId88g6NbrYuB7h4FvH+OPgMw5b1WWCPVxQAnwUSOh/lut9Q0tKUxCXbRGWVUNn207wVubjlFwrpKEcB+emNhPTqi2Rk2lceu/4hw4l21+zjH65YuzjWXncqG26uJnlCN49wbfvsbDJ+Lia88gmcGyHTQV6k1OwqG13qSUCrNoVUK0UlcXJx4YG8GdI3uzePsJ3th4lDve2caw3t7Mn9iPcf38JNyby8nVaIF7hzW8jdbGCJwz6XD6iPEoPGo8p2+B6rI6+3MD337QK9a4MrbXEOP2gc5d2vY4xCWa1aduDvWVjbTUlwGZQDZGq31/A/uZB8wDCA0NHZaRkdHauoUAjBOqXyRn8sb6I2QXVTA42Iv5E/tx9YBOehFTe9LaaNmfD/vTR6Eg1bhdYNkpYxvlaNwHNnCIEfKBcRL0V8giJ0qbCHVPwKS1LlFKTQVe1lr3a2qf0v0iLKmqxsSXOzN5df0RMs+UEx3oyeNX92PywJ6da/qBjkBrKM4ywj1n98Xn0gJj/fmgd/MCUzXUVhtTJNRWm9/X1FlebfTzhyRA2FjjNoP+Azp1N0+bh3o926YD8VrrU41tJ6Eu2kJ1rYmvdmXx+vojpJ8uY0BANx6/uh/XxXSyuWU6Gq2NfvrsXUbA5+w1um4cnMDRGRycwdHJ/Ox86fKKIjix1biBOEAXHwgbbYR879HGPPadKOTbo6UeAORprbVSKgFYCvTWTexYQl20pZpaEyv2ZvPquiMcLSilbw8PHr+6LzfEBtrXrJCdyZkMyPjR6MtP31wn5L2NcA8bY3Tv+Pc3ltkpS4x+WQQkAn5AHvAc4AygtX5TKfUY8DBQA5QDT2mttzZVmIS6aA+1Js2qfTm8si6Nw3klRPi58+iEvtw0JBAnx87TurNLl4T8Fjhb5xydR08j3P0H1HkeYMxlb+Pk4iMhAJPJmBVywbojHMwpJtSnK49O6MPNccH2c7OOzu7sSWMu+4JUKDh08bnuTUu6+Bjh7h1mjLdXjkZXj4MTODiaH3XfOxl/IHz6GMM33f2sfnGWhLoQdWit+eFgPgvWprEvq4ig7l14OLEPM+ODcXVytHZ5wtLOn7QtSIWCwxeDvuikcXL2wsN06fvLL8o6z9UTfMIvhryv+dknwriTVTsEvoS6EPXQWrPhcAEL1qax68RZAjzdeGh8BHMSQnFzlnDv9EwmY8rj2irjoqzCY8b4/MJjxtDNwmNGn76uvfgZR1djrL6jszGZmqOzcS3A+deOLhcfMTMg7s5WlXbFFx8JYY+UUkzo34PESH9+PHKaBWvTeH7FAV7bcJQHx0Vw+4hQ279Btmg9BwfAwQhjv77G43K11Uawnw/64ixjWW2l8cegtvric02dZdVFUF3eZqVLS10Is5+PGeG+9ehpfN1dmDsugl+N7I27q4S76Dik+0WIFkpKL2TBuiNsOlyAd1dn7h8Tzl2jwvB0c7Z2aUJIqAvRWrtPnuWVtWmsTc3H082Ju0eFcddVYfh3c7V2aaITk1AX4gqlZBXxyro0vjuQh7OjA7cMDeL+MRH07eFh7dJEJyShLoSFHCso4d0tx1manElljYlronowd2wECeE+MnmYaDcS6kJY2OmSSj7+OYOPfsqgsLSKwcFezB0XwZToALlKVbQ5CXUh2khFdS3LdmbyzubjHD9VSrB3F+4fE86s+BAZMSPajIS6EG3MZNL8cDCPhZuPsSP9DF5dnLn7qt7cPSoMXw85qSosS0JdiHa088QZ3tp4lDX783BzdmDO8FDmjosgqLvcFEJYhoS6EFZwJP8cb248xvJdWQBMGxLIw+P70K9nNytXJmydhLoQVpR9tpyFm4+xePtJyqtrmTSwJw8n9mFoqP3O9y3aloS6EB1AYWkVH25N54Ot6RSVVzMywoeHE/vKjbJFi0moC9GBlFbWsGj7Cd7ZfJzc4gqienkyd2w4N8QGyrzuolkk1IXogCpravl6VzYLNx8jLb+EAE837hkdxm0JoXh1kTlmRMMk1IXowEwmzca0AhZuOsbWo6dxd3FkTkIo944OI9i7q7XLEx2QhLoQNiIlq4h3Nh9j5d4cNDB1UC/mjg0nNri7tUsTHYiEuhA2JvtsOR9sTWfRthOcq6xhRLgPc8dGMGFADxwd5KRqZyehLoSNOldRzec7TvL+j+lknS0nxKcLd47ozaz4ELzdXaxdnrASCXUhbFx1rYnv9ufx0U/pbDteiKuTAzcNCeSuq8KICfKydnminUmoC2FHUnOL+einDL7amUV5dS1DQ7tz11VhXDcoAFcnuWF2ZyChLoQdKiqvZllyJh//nMHxU6X4ebgwZ3got48IJVDmmbFrEupC2DGTSbPlyCk++imdtan5OCjFlOgA7h8bLlMR2KmmQl0mfRbChjk4KMZF+jMu0p+ThWV88nMGi7af4Nt9OQzr7c0DY8KZHB0go2Y6EWmpC2FnSitr+CLpJO/9mM6JwjJCfLpw76hwZg0PwUNu3mHzpPtFiE6q1qT5/kAe72w+RlLGGbq5OXF7Qij3jA6jl5f0u9uqKw51pdR7wA1AvtY6pp71CngZmAqUAfdorXc2VZiEuhDtZ9eJM7y75TirU3JRwPWxvbh/jFytaoss0af+AfAq8FED668D+pkfI4A3zM9CiA4iLtSbV2/3JvNMGR/8mM7iHSf5enc2g0O6c8eIUG6MDaSLiwyJtAfN6n5RSoUBKxtoqb8FbNBaLzK/PwQkaq1zGtuntNSFsJ5zFcaQyE+2neBIfgmebk7cMiyYO0b0pm8PD2uXJxrRHqNfgoCTdd5nmpc1GupCCOvp5ubMPaPDuXtUGNuPF/LJthN88nMG7/+YzsgIH+4c2ZvJAwNkjncb1K6nwpVS84B5AKGhoe351UKIeiilGBHhy4gIX06VDGRJ0kk+23aCxz7bhZ+HK7OHBzNneCghPjINsK2Q7hchxCVqTZpNaQV8+nMG61Lz0UBipD+3j+jNhP7+ODlK692a2qP75RvgMaXUYowTpEVNBboQouNydFBM6N+DCf17kHW2nMXbT/D5jpPM/SiJAE83Zg0PYfbwEIJkOoIOqTlDGhcBiYAfkAc8BzgDaK3fNA9pfBWYgjGk8V6tdZNNcGmpC2E7ampNrEvN57PtJ9h4uACQ1ru1yMVHQgiLyjxTxuc7TvL5jpPkn6ukp6crs+NDmJ0QKq33diChLoRoE+db74u2n2BDndb7bQmhXD2gh7Te24iEuhCizWWeKWPJjpN8nnSSvGJpvbclCXUhRLuRvve2J6EuhLCKy/veZeSMZUioCyGsqqbWxFpz3/vGwwUoILF/D2YPD+HqAT1wltZ7i8hNMoQQVuXk6MC10QFcGx3AycIyliQZrfd1qfn4urswPS6ImfHBDAjwtHapdkFa6kKIdldTa2JTWgFLdmSyNjWP6lpNbLAXM4cFM21wEF5dna1dYocl3S9CiA6tsLSK5buy+CI5k4M5xbg4OTB5YE9mxocwpq+f3IrvMhLqQgibkZJVxNLkTJbvzuJsWTW9vNyYMTSIW4YGE+EvUwKDhLoQwgZV1tSy9mA+XySdZOPhAkwa4kK7c8vQYG6MDezU3TMS6kIIm5ZfXMHy3VksS87iUN45XJwcmBTVk1uGBTGuX+cb+y6hLoSwC1pr9mcXszQ5k2/2ZFNYWoWfhws3DTG6ZwYGdo7RMxLqQgi7U1VjYsOhfJbtzGRdaj7VtZqoXp5MHxLI9bG9CPa235t6SKgLIezamdIqVuzNZllyJnsyiwAYGtqdG2KNgO/p6WblCi1LQl0I0WlknC5l5d4cVu7N4WBOMUrB8DAfbhwcyHUxAfh5uFq7xCsmoS6E6JSO5Jewcm82K/fmcCS/BAcFo/r4cUNsL6bEBNC9q4u1S2wVCXUhRKemteZQ3jlW7slhxd5sMk6X4eSgGN3Xj+sH9WJydE+bCngJdSGEMNNak5JVzMp92azal8PJwvKLAR/bi2sHBnT4MfAS6kIIUQ+tNfuyivh2Xw7f7s0h84wR8GP6+TF1UMcNeAl1IYRowoWA35vDt/uMgHd2NFrwUwf1YvLAjtNFI6EuhBAtoLVmb2YRq/ZdDPjzXTRTBwUweWAA3u7WC3gJdSGEaKW6XTSr9+VyorAMRwfFqD6+RhdNdAA+7RzwEupCCGEB56cpWLUvh1X7ckg/bQT8VRG+XDfIuAlIe4yDl1AXQggL01pzIKeY1ftyWbUvh2OnSnFQEN/bh0kDezI5uie9fd3b5Lsl1IUQog1prUnNPcfqlFy+P5DHwZxiAPr37HYh4AcFeaGUZW72IaEuhBDt6GRhGd8fyOO7A7lsP16ISUOAp9uFgB8R7ouLU+unC5ZQF0IIKzlTWsW61Hy+O5DLxsMFVFSb6ObqxPyJ/Zg7LqJV+2wq1J1aXa0QQohGebu7cMuwYG4ZFkxFdS1b0k7x3YFcenVvu5kjJdSFEKIduDk7cs3AnlwzsGebfk+zOnaUUlOUUoeUUkeUUs/Us/4epVSBUmq3+fGA5UsVQgjRlCZb6kopR+A1YBKQCexQSn2jtT5w2aafa60fa4MahRBCNFNzWuoJwBGt9TGtdRWwGLipbcsSQgjRGs0J9SDgZJ33meZll7tFKbVXKbVUKRVS346UUvOUUklKqaSCgoJWlCuEEKIxrR8seakVQJjWOhb4Hviwvo201m9rreO11vH+/v4W+mohhBDnNSfUs4C6Le9g87ILtNantdaV5rfvAMMsU54QQoiWaE6o7wD6KaXClVIuwBzgm7obKKV61Xk7DThouRKFEEI0V5OjX7TWNUqpx4A1gCPwntZ6v1Lqj0CS1vobYL5SahpQAxQC97RhzUIIIRpgtWkClFIFQEYrP+4HnLJgOR2BvR2TvR0P2N8x2dvxgP0dU33H01tr3eBJSauF+pVQSiU1NveBLbK3Y7K34wH7OyZ7Ox6wv2NqzfFYavSLEEKIDkBCXQgh7Iithvrb1i6gDdjbMdnb8YD9HZO9HQ/Y3zG1+Hhssk9dCCFE/Wy1pS6EEKIeEupCCGFHbC7Um5rb3RYppdKVUvvMc9Hb3D3+lFLvKaXylVIpdZb5KKW+V0qlmZ+9rVljSzVwTM8rpbLq3DdgqjVrbAmlVIhSar1S6oBSar9S6gnzcpv8nRo5Hlv+jdyUUtuVUnvMx/QH8/JwpdQ2c+Z9br6yv+H92FKfunlu98PUmdsduK2eud1tilIqHYjXWtvkRRNKqXFACfCR1jrGvOzvQKHW+gXzH19vrfV/W7POlmjgmJ4HSrTW/7Rmba1hnsqjl9Z6p1KqG5AMTMe4+tvmfqdGjmcWtvsbKcBda12ilHIGtgBPAE8BX2qtFyul3gT2aK3faGg/ttZSl7ndOyCt9SaM6SHquomLs3V+iPEfnM1o4JhsltY6R2u90/z6HMb8TEHY6O/UyPHYLG0oMb91Nj80cDWw1Ly8yd/I1kK9uXO72xoNfKeUSlZKzbN2MRbSU2udY36dC7TtjRnbz2Pm+wa8ZytdFZdTSoUBccA27OB3uux4wIZ/I6WUo1JqN5CPMY35UeCs1rrGvEmTmWdroW6vxmithwLXAY+a/9ffbmijj892+vka9gbQBxgC5AD/smo1raCU8gCWAU9qrYvrrrPF36me47Hp30hrXau1HoIxxXkCMKCl+7C1UG9ybndbpLXOMj/nA19h/Ji2Lu/8lMzm53wr13PFtNZ55v/oTMBCbOx3MvfTLgM+1Vp/aV5ss79Tfcdj67/ReVrrs8B64Cqgu1Lq/Iy6TWaerYV6k3O72xqllLv5RA9KKXdgMpDS+KdswjfA3ebXdwNfW7EWi7jsvgE3Y0O/k/kk3LvAQa31i3VW2eTv1NDx2Phv5K+U6m5+3QVjQMhBjHC/1bxZk7+RTY1+ATAPUXqJi3O7/8W6FV0ZpVQERuscjPntP7O1Y1JKLQISMaYJzQOeA5YDS4BQjCmWZ2mtbebEYwPHlIjxv/UaSAcerNMf3aEppcYAm4F9gMm8+FmMfmib+50aOZ7bsN3fKBbjRKgjRoN7idb6j+aMWAz4ALuAO+vcae6X+7G1UBdCCNEwW+t+EUII0QgJdSGEsCMS6kIIYUck1IUQwo5IqAshhB2RUBdCCDsioS6EEHbk/wOhxrsdoYfkEgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.legend(['train','validation'])\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FYTG6GW_Ne0F",
        "outputId": "d3fd1f89-838b-4bc7-bd90-8d489238d92c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "313/313 [==============================] - 5s 17ms/step\n"
          ]
        }
      ],
      "source": [
        "# preds = (model.predict(testX.reshape((testX.shape[0],testX.shape[1]))) > 0.5).astype(\"int32\")\n",
        "#preds = model.predict(testX)\n",
        "preds = model.predict(testX.reshape((testX.shape[0],testX.shape[1])))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ntW3MlNiNsUB"
      },
      "outputs": [],
      "source": [
        "def get_word(n, tokenizer):\n",
        "      for word, index in tokenizer.word_index.items():\n",
        "          # print(word, index, n)\n",
        "          if index == n:\n",
        "              return word\n",
        "      return None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Jm02NQySLofN"
      },
      "outputs": [],
      "source": [
        "for word, index in eng_tokenizer.word_index.items():\n",
        "  print(word)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "754a7f79-302a-40dc-d734-1abf48a28936",
        "id": "jPD4x9qxDWQb"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 68%|█████████████████████████            | 6758/10000 [00:12<00:05, 580.84it/s]"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "from tqdm import tqdm\n",
        "preds_text = []\n",
        "for i in tqdm(preds):\n",
        "       temp = []\n",
        "       for j in range(len(i)):\n",
        "            token_index = np.argmax(i[j])\n",
        "            t = get_word(token_index, eng_tokenizer)\n",
        "            if j > 0:\n",
        "                prev_token_index = np.argmax(i[j-1])\n",
        "                if (t == get_word(prev_token_index, eng_tokenizer)) or (t == None):\n",
        "                     temp.append('')\n",
        "                else:\n",
        "                     temp.append(t)\n",
        "            else:\n",
        "                   if(t == None):\n",
        "                          temp.append('')\n",
        "                   else:\n",
        "#                           print(t)\n",
        "                          temp.append(t)\n",
        "\n",
        "       preds_text.append(' '.join(temp))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oHJSxzxJq2rR",
        "outputId": "443db9a0-991b-431f-daf2-fe1783a34999"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['tom asked a beer    ',\n",
              " 'tom kneeled      ',\n",
              " 'tomll will      ',\n",
              " 'show it to     ',\n",
              " 'you have a job    ',\n",
              " 'it was really fun    ',\n",
              " 'the sky was     ',\n",
              " 'tom isnt a adult    ',\n",
              " 'i bought the car    ',\n",
              " 'tom came alone     ',\n",
              " 'i love my book    ',\n",
              " 'i was cold     ',\n",
              " 'are you bored     ',\n",
              " 'they were very     ',\n",
              " 'tom ate      ',\n",
              " 'im surprised      ',\n",
              " 'please call it     ',\n",
              " 'tom is coughing     ',\n",
              " 'it was a bad    ',\n",
              " 'you cant stop     ',\n",
              " 'i took for my hat   ',\n",
              " 'tom made mary     ',\n",
              " 'tom spoke quietly     ',\n",
              " 'i dont like tom    ',\n",
              " 'those are      ',\n",
              " 'im not a enemy    ',\n",
              " 'tom stopped out     ',\n",
              " 'thats probably      ',\n",
              " 'i hate to     ',\n",
              " 'may i sit with you   ',\n",
              " 'he is good at man   ',\n",
              " 'tom is unfriendly     ',\n",
              " 'is this your wine    ',\n",
              " 'its not my style    ',\n",
              " 'tom was      ',\n",
              " 'tom gave me a cab   ',\n",
              " 'my headache are fat    ',\n",
              " 'just carefully      ',\n",
              " 'tom is vain     ',\n",
              " 'who your      ',\n",
              " 'they were lucky     ',\n",
              " 'i have a bicycle    ',\n",
              " 'im proud of you    ',\n",
              " 'its not a dream    ',\n",
              " 'its so sweet     ',\n",
              " 'you make me     ',\n",
              " 'you can do it    ',\n",
              " 'his dog is small    ',\n",
              " 'this book is     ',\n",
              " 'its really strange     ',\n",
              " 'thats toms      ',\n",
              " 'thats is house     ',\n",
              " 'i dont want to die   ',\n",
              " 'tom stood up     ',\n",
              " 'i thought as much    ',\n",
              " 'im as old as tom   ',\n",
              " 'tom is different     ',\n",
              " 'what are your plans    ',\n",
              " 'who went to     ',\n",
              " 'theyre very good     ',\n",
              " 'i think you know you   ',\n",
              " 'she is      ',\n",
              " 'im coming to     ',\n",
              " 'hes a good     ',\n",
              " 'i took the     ',\n",
              " 'is that really     ',\n",
              " 'i dont buy much    ',\n",
              " 'tom isnt too big    ',\n",
              " 'tom has money     ',\n",
              " 'close your eyes     ',\n",
              " 'we never help tom    ',\n",
              " 'tom wasnt stupid     ',\n",
              " 'i knew tom was it   ',\n",
              " 'i worked myself     ',\n",
              " 'please down one     ',\n",
              " 'hes teaches arabic     ',\n",
              " 'have a good trip    ',\n",
              " 'control yourself      ',\n",
              " 'tom is a  poet   ',\n",
              " 'im in a     ',\n",
              " 'tom was good     ',\n",
              " 'im almost about     ',\n",
              " 'we already did it    ',\n",
              " 'it started to snow    ',\n",
              " 'youre worried      ',\n",
              " 'do you like to    ',\n",
              " 'why is it so    ',\n",
              " 'tom was      ',\n",
              " 'tom made me     ',\n",
              " 'who wrote tom that    ',\n",
              " 'lets play lunch     ',\n",
              " 'theyll will it     ',\n",
              " 'does tom like anything    ',\n",
              " 'thats not      ',\n",
              " 'dont be so sure    ',\n",
              " 'tom wants to see us   ',\n",
              " 'excuse my clumsiness     ',\n",
              " 'we prayed french     ',\n",
              " 'the news grew me    ',\n",
              " 'tom was tipsy     ',\n",
              " 'tom was so hot    ',\n",
              " 'i have a with here   ',\n",
              " 'what do you     ',\n",
              " 'dont be insulted     ',\n",
              " 'thats quite sweet     ',\n",
              " 'he is up     ',\n",
              " 'tom is mean     ',\n",
              " 'can i help     ',\n",
              " 'tom hates to     ',\n",
              " 'you look confused     ',\n",
              " 'he the      ',\n",
              " 'tom is quite cute    ',\n",
              " 'she did him happy    ',\n",
              " 'this is really     ',\n",
              " 'show me for it    ',\n",
              " 'i am good sad    ',\n",
              " 'i said      ',\n",
              " 'it never work     ',\n",
              " 'tom is your father    ',\n",
              " 'we met      ',\n",
              " 'we live in boston    ',\n",
              " 'tom is outrageous     ',\n",
              " 'i know its     ',\n",
              " 'i love   all   ',\n",
              " 'tom disappointed me     ',\n",
              " 'yours is over there    ',\n",
              " 'tom loves something     ',\n",
              " 'everyone were scared     ',\n",
              " 'its is raining     ',\n",
              " 'when did you return    ',\n",
              " 'hell will be to    ',\n",
              " 'no was      ',\n",
              " 'who hit tom     ',\n",
              " 'tom looks naive     ',\n",
              " 'do you like coffee    ',\n",
              " 'dont say that     ',\n",
              " 'tom seemed amazing     ',\n",
              " 'thats very      ',\n",
              " 'dont laugh      ',\n",
              " 'the room is full    ',\n",
              " 'i was very angry    ',\n",
              " 'hold the candle     ',\n",
              " 'did you talk with    ',\n",
              " 'it was a small    ',\n",
              " 'you cant go     ',\n",
              " 'tom wasnt crying     ',\n",
              " 'i have to a job   ',\n",
              " 'itll be      ',\n",
              " 'tom is so happy    ',\n",
              " 'do you need a car   ',\n",
              " 'we both too     ',\n",
              " 'tom came here     ',\n",
              " 'i have a pacemaker    ',\n",
              " 'lets us a     ',\n",
              " 'are you busy     ',\n",
              " 'its a bit case    ',\n",
              " 'tom wasnt crying     ',\n",
              " 'i love  you    ',\n",
              " 'tom started to     ',\n",
              " 'i dont know tom    ',\n",
              " 'they let me     ',\n",
              " 'wash up      ',\n",
              " 'this story is true    ',\n",
              " 'tom is slowly     ',\n",
              " 'im am really stupid    ',\n",
              " 'please up please     ',\n",
              " 'tom isnt home     ',\n",
              " 'i was hungry     ',\n",
              " 'tom and im well    ',\n",
              " 'i need a advice    ',\n",
              " 'why isnt that good    ',\n",
              " 'my mother is mad    ',\n",
              " 'do you a drink    ',\n",
              " 'theyll find us     ',\n",
              " 'i know tom is tom   ',\n",
              " 'give it half to    ',\n",
              " 'tom has recovered     ',\n",
              " 'god       ',\n",
              " 'i never over there    ',\n",
              " 'im in      ',\n",
              " 'thats is      ',\n",
              " 'tom likes that     ',\n",
              " 'keep the room     ',\n",
              " 'keep it      ',\n",
              " 'he has a rude memory   ',\n",
              " 'it do me     ',\n",
              " 'she took in a    ',\n",
              " 'this is a big    ',\n",
              " 'tom made us do it   ',\n",
              " 'i just real     ',\n",
              " 'do you have a bike   ',\n",
              " 'thats my cd     ',\n",
              " 'i love mahjong     ',\n",
              " 'youre not that old    ',\n",
              " 'i didnt mean you    ',\n",
              " 'i didnt deserve that    ',\n",
              " 'we did it     ',\n",
              " 'the time won     ',\n",
              " 'everyone was hungry     ',\n",
              " 'thats is the     ',\n",
              " 'i dont like doctors    ',\n",
              " 'tom fed mary     ',\n",
              " 'i never to     ',\n",
              " 'im not happy     ',\n",
              " 'wait us      ',\n",
              " 'thats isnt good     ',\n",
              " 'tom never      ',\n",
              " 'have you eaten     ',\n",
              " 'i study      ',\n",
              " 'how do tom do this   ',\n",
              " 'tom wasnt mean     ',\n",
              " 'are you      ',\n",
              " 'he began to cry    ',\n",
              " 'he didnt say to lie   ',\n",
              " 'tom talks a lot    ',\n",
              " 'hes studying      ',\n",
              " 'tom isnt no hero    ',\n",
              " 'he has wrong     ',\n",
              " 'i have to  now   ',\n",
              " 'did you tell tom why   ',\n",
              " 'tom is waiting     ',\n",
              " 'theyre disgusting      ',\n",
              " 'it was a good mistake   ',\n",
              " 'im going with tom    ',\n",
              " 'tom isnt afraid     ',\n",
              " 'im ready      ',\n",
              " 'drive more      ',\n",
              " 'do you have rice    ',\n",
              " 'i did it for you   ',\n",
              " 'tom has a dog    ',\n",
              " 'how good is you    ',\n",
              " 'tom sat to the    ',\n",
              " 'tom knew a  lot   ',\n",
              " 'i met tom yesterday    ',\n",
              " 'i never said     ',\n",
              " 'she is young     ',\n",
              " 'do you have fever    ',\n",
              " 'you may enter now    ',\n",
              " 'no one knows why    ',\n",
              " 'you made tom cry    ',\n",
              " 'did you have my pen   ',\n",
              " 'just check it     ',\n",
              " 'is the tv     ',\n",
              " 'tom was a teacher    ',\n",
              " 'tom was there     ',\n",
              " 'thats a order     ',\n",
              " 'i am sick     ',\n",
              " 'i just him     ',\n",
              " 'tom knows it     ',\n",
              " 'tell tom about it    ',\n",
              " 'i want my money    ',\n",
              " 'shes is a child    ',\n",
              " 'tomll will      ',\n",
              " 'tom is single     ',\n",
              " 'hes is      ',\n",
              " 'im a debt     ',\n",
              " 'theyre all excited     ',\n",
              " 'are you safe     ',\n",
              " 'arent not excited     ',\n",
              " 'hes not  hero    ',\n",
              " 'you can count at on   ',\n",
              " 'you can have it    ',\n",
              " 'that like that     ',\n",
              " 'what motivated you     ',\n",
              " 'she is fine     ',\n",
              " 'did you bring it    ',\n",
              " 'break it      ',\n",
              " 'it be happen     ',\n",
              " 'no one lives here    ',\n",
              " 'did you sue     ',\n",
              " 'are you happy     ',\n",
              " 'you cant fire me    ',\n",
              " 'they took      ',\n",
              " 'i can be too    ',\n",
              " 'men went cry     ',\n",
              " 'were are      ',\n",
              " 'i have a feet    ',\n",
              " 'your bath is ready    ',\n",
              " 'tom got in bed    ',\n",
              " 'he was wrong     ',\n",
              " 'toms is      ',\n",
              " 'i bought a new tie   ',\n",
              " 'you joking      ',\n",
              " 'i me to  me   ',\n",
              " 'where my      ',\n",
              " 'ill go by     ',\n",
              " 'you must believe me    ',\n",
              " 'my hurt      ',\n",
              " 'tom had a cold    ',\n",
              " 'we did it     ',\n",
              " 'this is      ',\n",
              " 'im not your slave    ',\n",
              " 'i loved her     ',\n",
              " 'im hungry      ',\n",
              " 'i loved that book    ',\n",
              " 'its quite      ',\n",
              " 'our cat is empty    ',\n",
              " 'how did he do that   ',\n",
              " 'it was depressing     ',\n",
              " 'this is my dad    ',\n",
              " 'is this too taken    ',\n",
              " 'this isnt my     ',\n",
              " 'im am      ',\n",
              " 'it was naive     ',\n",
              " 'you will die     ',\n",
              " 'were astonished      ',\n",
              " 'im is possible     ',\n",
              " 'tom isnt go alone    ',\n",
              " 'i dont like it    ',\n",
              " 'whos is today     ',\n",
              " 'are you working     ',\n",
              " 'give that to  me   ',\n",
              " 'tom helped me     ',\n",
              " 'i feel a news    ',\n",
              " 'my mother is full    ',\n",
              " 'i worked fine     ',\n",
              " 'let play to 230    ',\n",
              " 'this is very bad    ',\n",
              " 'i saw you saw    ',\n",
              " 'tom has many cats    ',\n",
              " 'he loves  fish    ',\n",
              " 'weve were promoted     ',\n",
              " 'this is for     ',\n",
              " 'i me      ',\n",
              " 'how does      ',\n",
              " 'dont look them     ',\n",
              " 'i wanted to go home   ',\n",
              " 'we wont forget you    ',\n",
              " 'have a good trip    ',\n",
              " 'what tom      ',\n",
              " 'tom always well     ',\n",
              " 'i like to help you   ',\n",
              " 'call your weapons     ',\n",
              " 'i think tom is in   ',\n",
              " 'do you know that    ',\n",
              " 'ill will tom     ',\n",
              " 'tom wasnt so much    ',\n",
              " 'tom took a flat    ',\n",
              " 'they were great     ',\n",
              " 'she has few books    ',\n",
              " 'his bit      ',\n",
              " 'tom is in a light   ',\n",
              " 'who do      ',\n",
              " 'seriously you      ',\n",
              " 'i was lonely     ',\n",
              " 'what did they say    ',\n",
              " 'read it wild     ',\n",
              " 'i live in a city   ',\n",
              " 'i can wait     ',\n",
              " 'wheres the      ',\n",
              " 'its is problem     ',\n",
              " 'i made that way tom   ',\n",
              " 'do i want to i you  ',\n",
              " 'ill going to     ',\n",
              " 'tom is a lawyer    ',\n",
              " 'tom didnt come     ',\n",
              " 'i can see it    ',\n",
              " 'i really fine     ',\n",
              " 'im am your father    ',\n",
              " 'i never met him    ',\n",
              " 'we want tom     ',\n",
              " 'do you      ',\n",
              " 'theyre unreliable      ',\n",
              " 'read it      ',\n",
              " 'im really grateful     ',\n",
              " 'he gets at early    ',\n",
              " 'tom is scary     ',\n",
              " 'thanks for      ',\n",
              " 'i took tom     ',\n",
              " 'i like      ',\n",
              " 'i want a     ',\n",
              " 'you are a student    ',\n",
              " 'where have you     ',\n",
              " 'dont worry about that    ',\n",
              " 'my  gone here    ',\n",
              " 'may i help you    ',\n",
              " 'it do      ',\n",
              " 'it may be     ',\n",
              " 'it was freezing     ',\n",
              " 'i cant eat the    ',\n",
              " 'tom lost his     ',\n",
              " 'tom was very slow    ',\n",
              " 'toms car is     ',\n",
              " 'tom was excited     ',\n",
              " 'i never met to    ',\n",
              " 'everyone knows      ',\n",
              " 'i fell up     ',\n",
              " 'i just feel     ',\n",
              " 'you look like a    ',\n",
              " 'was it too     ',\n",
              " 'nobody saw us     ',\n",
              " 'it wasnt so     ',\n",
              " 'tom cooked that     ',\n",
              " 'whats in the     ',\n",
              " 'do you read a lot   ',\n",
              " 'tom seemed worried     ',\n",
              " 'thats a order     ',\n",
              " 'you need a job    ',\n",
              " 'i have a big dog   ',\n",
              " 'do you want a    ',\n",
              " 'is for with on    ',\n",
              " 'tom went gone     ',\n",
              " 'i deleted the file    ',\n",
              " 'tom and mary     ',\n",
              " 'i dont like to    ',\n",
              " 'i ran up at    ',\n",
              " 'do you remember anything    ',\n",
              " 'no one else     ',\n",
              " 'ill go with tom    ',\n",
              " 'italy is baby sheep    ',\n",
              " 'we want to  why   ',\n",
              " 'tom isnt happy     ',\n",
              " 'dont be rude     ',\n",
              " 'is it good     ',\n",
              " 'the boys are thirsty    ',\n",
              " 'i like your the    ',\n",
              " 'did  that     ',\n",
              " 'whose idea was it    ',\n",
              " 'do you like oranges    ',\n",
              " 'youre young      ',\n",
              " 'take your homework     ',\n",
              " 'how rude      ',\n",
              " 'tom is himself     ',\n",
              " 'im hiding tom     ',\n",
              " 'toms boat sank     ',\n",
              " 'when will you come    ',\n",
              " 'i wasnt fall     ',\n",
              " 'you believed tom     ',\n",
              " 'dont play up     ',\n",
              " 'he went in ball    ',\n",
              " 'where was been     ',\n",
              " 'tom kept smiled     ',\n",
              " 'i did what to saw   ',\n",
              " 'i can do it that   ',\n",
              " 'im home at early    ',\n",
              " 'is that love     ',\n",
              " 'you have to go    ',\n",
              " 'i cant do that    ',\n",
              " 'is that a yes    ',\n",
              " 'you have time     ',\n",
              " 'tell tom i called    ',\n",
              " 'tom was relieved     ',\n",
              " 'i almost confused naked    ',\n",
              " 'he married my head    ',\n",
              " 'i miss my family    ',\n",
              " 'we cant let tom    ',\n",
              " 'we were there     ',\n",
              " 'everyone worried      ',\n",
              " 'tom is my dad    ',\n",
              " 'this has to     ',\n",
              " 'that was close     ',\n",
              " 'i cant swim at    ',\n",
              " 'i like to  him   ',\n",
              " 'tom is a boy scout   ',\n",
              " 'tom isnt creepy     ',\n",
              " 'tom was a  hero   ',\n",
              " 'he began to cry    ',\n",
              " 'i want to see you   ',\n",
              " 'what do you need    ',\n",
              " 'tom has a ring    ',\n",
              " 'mary is toms sister    ',\n",
              " 'how did you do it   ',\n",
              " 'wheres your dog     ',\n",
              " 'where is this     ',\n",
              " 'this desk is it    ',\n",
              " 'get out of my me   ',\n",
              " 'they said it     ',\n",
              " 'tom was belligerent     ',\n",
              " 'who gave you that    ',\n",
              " 'its very  big    ',\n",
              " 'tom worked hard     ',\n",
              " 'no one can true    ',\n",
              " 'everyone lying      ',\n",
              " 'tom isnt ready     ',\n",
              " 'tom cant answer     ',\n",
              " 'i dancing      ',\n",
              " 'lets play inside     ',\n",
              " 'theyre strong      ',\n",
              " 'they hugged      ',\n",
              " 'i wont play a    ',\n",
              " 'please come to me    ',\n",
              " 'were upstairs      ',\n",
              " 'i was      ',\n",
              " 'whats new      ',\n",
              " 'i need a lot job   ',\n",
              " 'it cant be true    ',\n",
              " 'i want a lawyer    ',\n",
              " 'why are you so    ',\n",
              " 'youre a good man    ',\n",
              " 'i want an guitar    ',\n",
              " 'are you      ',\n",
              " 'leave it to me    ',\n",
              " 'put up the     ',\n",
              " 'lets play it     ',\n",
              " 'get me my newspaper    ',\n",
              " 'i do the road    ',\n",
              " 'get lost      ',\n",
              " 'wow       ',\n",
              " 'tom started it     ',\n",
              " 'the is      ',\n",
              " 'i worked in a bank   ',\n",
              " 'tom is still angry    ',\n",
              " 'tom came out     ',\n",
              " 'tom and mary     ',\n",
              " 'is she a flight    ',\n",
              " 'are they      ',\n",
              " 'she is a     ',\n",
              " 'tom waved mary     ',\n",
              " 'my feel still hurts    ',\n",
              " 'im not do that    ',\n",
              " 'tom is unbelievable     ',\n",
              " 'hes a  jealous    ',\n",
              " 'i want a map    ',\n",
              " 'no one it     ',\n",
              " 'tom kept his word    ',\n",
              " 'read this      ',\n",
              " 'tom noticed it     ',\n",
              " 'tom is you     ',\n",
              " 'tom is a hermit    ',\n",
              " 'its a  fact    ',\n",
              " 'they made it     ',\n",
              " 'tom is my brother    ',\n",
              " 'its just a joke    ',\n",
              " 'i suppressed a yawn    ',\n",
              " 'is this your wine    ',\n",
              " 'he was in at    ',\n",
              " 'do you like that    ',\n",
              " 'lets talk      ',\n",
              " 'shut door      ',\n",
              " 'i like this book    ',\n",
              " 'tom is dejected     ',\n",
              " 'i wasnt hungry     ',\n",
              " 'get away      ',\n",
              " 'how much did you have   ',\n",
              " 'this ones is mine    ',\n",
              " 'keep your eyes     ',\n",
              " 'let me to     ',\n",
              " 'get out a bed    ',\n",
              " 'im in the attic    ',\n",
              " 'thats a an case    ',\n",
              " 'i have the money    ',\n",
              " 'i cant do that    ',\n",
              " 'tom has a headache    ',\n",
              " 'whats this      ',\n",
              " 'keep up mouth     ',\n",
              " 'tom asked mary     ',\n",
              " 'we need more     ',\n",
              " 'i dont order     ',\n",
              " 'tom is outrageous     ',\n",
              " 'everybody likes me     ',\n",
              " 'we get talk us    ',\n",
              " 'bring a hat     ',\n",
              " 'give me a favor    ',\n",
              " 'shes not a doctor    ',\n",
              " 'seriously       ',\n",
              " 'tom behaved quickly     ',\n",
              " 'did i touch a nerve   ',\n",
              " 'go for a walk    ',\n",
              " 'she is a good heart   ',\n",
              " 'never trust anybody     ',\n",
              " 'who wants to     ',\n",
              " 'this isnt about tom    ',\n",
              " 'how is  this    ',\n",
              " 'thats is      ',\n",
              " 'tom is prejudiced     ',\n",
              " 'you look very much    ',\n",
              " 'do you like to    ',\n",
              " 'tom felt lonely     ',\n",
              " 'is live dying     ',\n",
              " 'eat your vegetables     ',\n",
              " 'tom fed the sheep    ',\n",
              " 'i was a     ',\n",
              " 'he cant say trusted    ',\n",
              " 'tom wasnt smiled     ',\n",
              " 'im going you     ',\n",
              " 'do you like anything    ',\n",
              " 'tom is outrageous     ',\n",
              " 'dont go up     ',\n",
              " 'tom is outrageous     ',\n",
              " 'does tom just tom    ',\n",
              " 'im not a owner    ',\n",
              " 'they lied me     ',\n",
              " 'ask him his name    ',\n",
              " 'theyre all here     ',\n",
              " 'its immoral      ',\n",
              " 'i didnt want that    ',\n",
              " 'ill take tom     ',\n",
              " 'speak       ',\n",
              " 'we have in time    ',\n",
              " 'tom talks fast     ',\n",
              " 'tom knows this     ',\n",
              " 'thats a order     ',\n",
              " 'she shot him     ',\n",
              " 'am i in the    ',\n",
              " 'im in  boston    ',\n",
              " 'i understand      ',\n",
              " 'im indebted the war    ',\n",
              " 'wheres the      ',\n",
              " 'i cant talk to tom   ',\n",
              " 'im leaving outside     ',\n",
              " 'did you know that    ',\n",
              " 'i ran the     ',\n",
              " 'theyre arent here     ',\n",
              " 'just me      ',\n",
              " 'what tom  done    ',\n",
              " 'tom is very     ',\n",
              " 'tom made it     ',\n",
              " 'i have epilepsy     ',\n",
              " 'read your coffee     ',\n",
              " 'it looks very good    ',\n",
              " 'didnt you see that    ',\n",
              " 'who wrote the bread    ',\n",
              " 'keep the      ',\n",
              " 'tom looks hopeful     ',\n",
              " 'i have a pacemaker    ',\n",
              " 'dont lie      ',\n",
              " 'were us      ',\n",
              " 'i have a glasses    ',\n",
              " 'i must to get out   ',\n",
              " 'try some      ',\n",
              " 'who wrote the car    ',\n",
              " 'youre drenched      ',\n",
              " 'tom has a mustache    ',\n",
              " 'i love korean     ',\n",
              " 'im am breakfast     ',\n",
              " 'isnt it great     ',\n",
              " 'theres no      ',\n",
              " 'did you get at up   ',\n",
              " 'he made to  it   ',\n",
              " 'were are      ',\n",
              " 'you cant lose     ',\n",
              " 'who talked      ',\n",
              " 'im a      ',\n",
              " 'can we afford me    ',\n",
              " 'its so      ',\n",
              " 'read the      ',\n",
              " 'is your      ',\n",
              " 'i like your love    ',\n",
              " 'lets play      ',\n",
              " 'they have      ',\n",
              " 'why you  so    ',\n",
              " 'i cant say that    ',\n",
              " 'were on next     ',\n",
              " 'let me handle that    ',\n",
              " 'thats not      ',\n",
              " 'tom stood there     ',\n",
              " 'im am      ',\n",
              " 'i was very crying    ',\n",
              " 'why do you do that   ',\n",
              " 'get me a     ',\n",
              " 'i like to eat    ',\n",
              " 'ill me  that    ',\n",
              " 'have a nice day    ',\n",
              " 'throw the dice     ',\n",
              " 'ill be careful     ',\n",
              " 'theyre are      ',\n",
              " 'go with tom     ',\n",
              " 'i toms  teeth    ',\n",
              " 'youve wore a hair    ',\n",
              " 'did tom kiss you    ',\n",
              " 'we arrested tom     ',\n",
              " 'he didnt not     ',\n",
              " 'do you live in    ',\n",
              " 'tom was blind     ',\n",
              " 'your eyes black     ',\n",
              " 'did you ever     ',\n",
              " 'tom cant cook     ',\n",
              " 'keep down      ',\n",
              " 'ill take dishes     ',\n",
              " 'its was your     ',\n",
              " 'tom made me for    ',\n",
              " 'tom disappointed me     ',\n",
              " 'i dont feel him    ',\n",
              " 'we have our money    ',\n",
              " 'i cant say that    ',\n",
              " 'i was born with tom   ',\n",
              " 'get a quickly     ',\n",
              " 'have your work     ',\n",
              " 'i wasnt like it    ',\n",
              " 'why i  come    ',\n",
              " 'i dont get to    ',\n",
              " 'thats is my dream    ',\n",
              " 'i on  monday    ',\n",
              " 'tom is coughing     ',\n",
              " 'they were grateful     ',\n",
              " 'tom cant swim     ',\n",
              " 'you look good     ',\n",
              " 'they were both     ',\n",
              " 'tom needs that     ',\n",
              " 'this car is you    ',\n",
              " 'the hot is hot    ',\n",
              " 'im a      ',\n",
              " 'my eyes      ',\n",
              " 'we didnt met     ',\n",
              " 'itll be in     ',\n",
              " 'god       ',\n",
              " 'its so mine     ',\n",
              " 'tom is      ',\n",
              " 'youre incredible      ',\n",
              " 'tom looks naive     ',\n",
              " 'stop squabbling      ',\n",
              " 'she scare to     ',\n",
              " 'tom is a tall man   ',\n",
              " 'they were drowned     ',\n",
              " 'tom is a chatterbox    ',\n",
              " 'tom ate pizza     ',\n",
              " 'i made it for    ',\n",
              " 'tom is a hermit    ',\n",
              " 'im going fine evening    ',\n",
              " 'tom had a cat    ',\n",
              " 'tom didnt stand     ',\n",
              " 'is that a news    ',\n",
              " 'are you      ',\n",
              " 'i dont like a city   ',\n",
              " 'that  ice     ',\n",
              " 'i think tom did it   ',\n",
              " 'tom chuckled      ',\n",
              " 'were very drunk     ',\n",
              " 'dont the  bus    ',\n",
              " 'be brave      ',\n",
              " 'is tom a doctor    ',\n",
              " 'he did that wrong    ',\n",
              " 'lets start here     ',\n",
              " 'this is my dog    ',\n",
              " 'lifes not fair     ',\n",
              " 'whats this      ',\n",
              " 'tom kept calm     ',\n",
              " 'is tom still in    ',\n",
              " 'what this      ',\n",
              " 'try it a more    ',\n",
              " 'were winning      ',\n",
              " 'tom is improving     ',\n",
              " 'she is      ',\n",
              " 'tom has braces     ',\n",
              " 'tom may be here    ',\n",
              " 'tighten the screws     ',\n",
              " 'it may be     ',\n",
              " 'tom is old     ',\n",
              " 'tom is      ',\n",
              " 'do you have a    ',\n",
              " 'mary is married     ',\n",
              " 'did you get up work   ',\n",
              " 'i have a map    ',\n",
              " 'tomll will      ',\n",
              " 'i think i did it   ',\n",
              " 'tom is a     ',\n",
              " 'keep carefully      ',\n",
              " 'its very nice     ',\n",
              " 'she is wrong     ',\n",
              " 'tom is up boston    ',\n",
              " 'tom seems smart     ',\n",
              " 'i like i like    ',\n",
              " 'are those your are    ',\n",
              " 'you cant go here    ',\n",
              " 'he has a headache    ',\n",
              " 'ill do that     ',\n",
              " 'thats is      ',\n",
              " 'im toms      ',\n",
              " 'thats not easy     ',\n",
              " 'they love      ',\n",
              " 'im a      ',\n",
              " 'its a good     ',\n",
              " 'i am a student    ',\n",
              " 'he has his     ',\n",
              " 'im toms  father    ',\n",
              " 'i cant swim     ',\n",
              " 'are you blind     ',\n",
              " 'i miss tom     ',\n",
              " 'tom was really good    ',\n",
              " 'ill will outside     ',\n",
              " 'im am toms     ',\n",
              " 'she is like a bad   ',\n",
              " 'why are you at home   ',\n",
              " 'i feel      ',\n",
              " 'go  with a    ',\n",
              " 'they was      ',\n",
              " 'i am a you    ',\n",
              " 'did we wake up    ',\n",
              " 'i almost knew it    ',\n",
              " 'i got here     ',\n",
              " 'im am      ',\n",
              " 'im a  blonde    ',\n",
              " 'youre unreliable      ',\n",
              " 'she went up     ',\n",
              " 'you have many eyes    ',\n",
              " 'tom killed mary     ',\n",
              " 'i have a idea    ',\n",
              " 'well kill you     ',\n",
              " 'i felt so of    ',\n",
              " 'dont trust anybody     ',\n",
              " 'is this your car    ',\n",
              " 'its over      ',\n",
              " 'im am on     ',\n",
              " 'i met to for help   ',\n",
              " 'do you like wine    ',\n",
              " 'i know im is    ',\n",
              " 'tom didnt help us    ',\n",
              " 'have another neighbor     ',\n",
              " 'wheres my      ',\n",
              " 'ill go by car    ',\n",
              " 'were ready for     ',\n",
              " 'tom was      ',\n",
              " 'i dont like doctors    ',\n",
              " 'i cant do that    ',\n",
              " 'someone singing      ',\n",
              " 'how was is the lake   ',\n",
              " 'take that      ',\n",
              " 'it was the     ',\n",
              " 'its going to     ',\n",
              " 'come closer      ',\n",
              " 'listen       ',\n",
              " 'we cant tell that tom   ',\n",
              " 'nothing stopped      ',\n",
              " 'was tom there     ',\n",
              " 'why is you so    ',\n",
              " 'tom has three cats    ',\n",
              " 'weve have enough     ',\n",
              " 'can you read     ',\n",
              " 'i will swim     ',\n",
              " 'dont worry yourself     ',\n",
              " 'i paid today     ',\n",
              " 'im me a message    ',\n",
              " 'read it      ',\n",
              " 'im a waiter     ',\n",
              " 'we all know it    ',\n",
              " 'i asked tom to live   ',\n",
              " 'dont make me     ',\n",
              " 'you dont lie     ',\n",
              " 'do you smoke a lot   ',\n",
              " 'he speaks english     ',\n",
              " 'come out it again    ',\n",
              " 'im rich      ',\n",
              " 'i heard her crying    ',\n",
              " 'the water is hot    ',\n",
              " 'you cant fire me    ',\n",
              " 'tom is quite weak    ',\n",
              " 'it was worse     ',\n",
              " 'the water was empty    ',\n",
              " 'go       ',\n",
              " 'ok to      ',\n",
              " 'i have to help tom   ',\n",
              " 'maybe may be wrong    ',\n",
              " 'tom was      ',\n",
              " 'i want to sleep abroad   ',\n",
              " 'tom listens      ',\n",
              " 'tom was relaxed     ',\n",
              " 'just us a     ',\n",
              " 'he has a good memory   ',\n",
              " 'where are you     ',\n",
              " 'tom has a stroke    ',\n",
              " 'dont listen      ',\n",
              " 'keep me posted     ',\n",
              " 'keep cool      ',\n",
              " 'tom died last class    ',\n",
              " 'tom is      ',\n",
              " 'im not your enemy    ',\n",
              " 'tom studied hockey     ',\n",
              " 'it isnt fun     ',\n",
              " 'are you a of    ',\n",
              " 'did you yell at tom   ',\n",
              " 'tom isnt naive     ',\n",
              " 'i hair long hair    ',\n",
              " 'tom felt sad     ',\n",
              " 'whats next      ',\n",
              " 'i was english     ',\n",
              " 'im take tom     ',\n",
              " 'i know its a    ',\n",
              " 'wait with  tom    ',\n",
              " 'im toms cousin     ',\n",
              " 'how do i get one   ',\n",
              " 'its a hard     ',\n",
              " 'may i sit a to   ',\n",
              " 'i said youd get a   ',\n",
              " 'just it to tom    ',\n",
              " 'may i use your pen   ',\n",
              " 'let the knife     ',\n",
              " 'tom has many talents    ',\n",
              " 'he came at bed    ',\n",
              " 'he has a shoe    ',\n",
              " 'everyone knows that     ',\n",
              " 'do you like running    ',\n",
              " 'tom works himself     ',\n",
              " 'do you like anything    ',\n",
              " 'tom is patient     ',\n",
              " 'when can we stop    ',\n",
              " 'im a therapist     ',\n",
              " 'i tried to save you   ',\n",
              " 'youre really sweet     ',\n",
              " 'do you have a minute   ',\n",
              " 'read these      ',\n",
              " 'this is      ',\n",
              " 'i have too a    ',\n",
              " 'i read a book    ',\n",
              " 'it looks like     ',\n",
              " 'dont look up     ',\n",
              " 'he likes to a    ',\n",
              " 'tom doesnt do that    ',\n",
              " 'you must to move    ',\n",
              " 'lets go in bed way   ',\n",
              " 'thats too to     ',\n",
              " 'tom is my boy    ',\n",
              " 'i have big teacher    ',\n",
              " 'everyone is      ',\n",
              " 'did you know this    ',\n",
              " 'they dont my idea    ',\n",
              " 'whats a nice     ',\n",
              " 'we want to go    ',\n",
              " 'i gave tom a cookie   ',\n",
              " 'tom is      ',\n",
              " 'dont you want to go   ',\n",
              " 'let him out     ',\n",
              " 'is  difficult     ',\n",
              " 'no one      ',\n",
              " 'does tom have a    ',\n",
              " 'whos not      ',\n",
              " 'i happy      ',\n",
              " 'we are at     ',\n",
              " 'stand aside please     ',\n",
              " 'does that work     ',\n",
              " 'he gave me a word   ',\n",
              " 'you need a car    ',\n",
              " 'thats is      ',\n",
              " 'tom is a  beard   ',\n",
              " 'i love her     ',\n",
              " 'i made it     ',\n",
              " 'tom is stubborn     ',\n",
              " 'i didnt kiss tom    ',\n",
              " 'cats is expensive     ',\n",
              " 'its time to     ',\n",
              " 'it cant be true    ',\n",
              " 'tom sat to     ',\n",
              " 'i was won     ',\n",
              " 'i know your language    ',\n",
              " 'tom didnt help me    ',\n",
              " 'who wrote this poem    ',\n",
              " 'is your home     ',\n",
              " 'i did what to saw   ',\n",
              " 'this apple is very    ',\n",
              " 'ill be over it    ',\n",
              " 'tom didnt like     ',\n",
              " 'where do you work    ',\n",
              " 'are you a of    ',\n",
              " 'did tom come     ',\n",
              " 'ill be to at    ',\n",
              " 'i dont expect you    ',\n",
              " 'i have here     ',\n",
              " 'mary can swim well    ',\n",
              " 'youre are no     ',\n",
              " 'im a bit     ',\n",
              " 'tom has blue eyes    ',\n",
              " 'its very      ',\n",
              " 'i was sick     ',\n",
              " 'the is  good    ',\n",
              " 'why didnt you know    ',\n",
              " 'maybe tom be     ',\n",
              " 'it looks      ',\n",
              " 'were in  danger    ',\n",
              " 'i just see well    ',\n",
              " 'her child is big    ',\n",
              " 'what are you     ',\n",
              " 'thats is      ',\n",
              " 'were nervous      ',\n",
              " 'im came with you    ',\n",
              " 'tom is a  guy   ',\n",
              " 'no one can  tom   ',\n",
              " 'i dont do that    ',\n",
              " 'just check there     ',\n",
              " 'please shut up     ',\n",
              " 'tom ate abroad     ',\n",
              " 'tom was eats     ',\n",
              " 'my son is a 10   ',\n",
              " 'toms is      ',\n",
              " 'this is your     ',\n",
              " 'isnt that a good    ',\n",
              " 'its is car     ',\n",
              " 'tom has a 9to5 bear   ',\n",
              " 'tom looked cried     ',\n",
              " 'its your home     ',\n",
              " 'tom hair his hair cut   ',\n",
              " 'try it on     ',\n",
              " 'who cheated      ',\n",
              " 'you cant go yet    ',\n",
              " 'tom kept on     ',\n",
              " 'no saw tom     ',\n",
              " 'tom is snoring     ',\n",
              " 'when will we go    ',\n",
              " 'i was at the library   ',\n",
              " 'tom is again     ',\n",
              " 'i think im good    ',\n",
              " 'take your kids     ',\n",
              " 'im glad youre see you   ',\n",
              " 'tom took the apple    ',\n",
              " 'i was been     ',\n",
              " 'turn up the radio    ',\n",
              " 'we ran our of    ',\n",
              " 'thats toms      ',\n",
              " 'tom is a chef    ',\n",
              " 'i sorry to     ',\n",
              " 'i hear      ',\n",
              " 'were eat water     ',\n",
              " ...]"
            ]
          },
          "execution_count": 61,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "preds_text"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 2.5 Verifying the results"
      ],
      "metadata": {
        "id": "t5toFXS4Bwjy"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DvuQiDPqqTS1"
      },
      "outputs": [],
      "source": [
        "pred_df = pd.DataFrame({'German Input' : test[:, 1], 'Predicted' : preds_text, 'Expected': test[:, 0]})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "KEi7lJz0qin7",
        "outputId": "29b9410f-d88b-48a9-e974-b36060fb0021"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>German Input</th>\n",
              "      <th>Predicted</th>\n",
              "      <th>Expected</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>tom hat ein bier bestellt</td>\n",
              "      <td>tom asked a beer</td>\n",
              "      <td>tom ordered a beer</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>tom wählte</td>\n",
              "      <td>tom kneeled</td>\n",
              "      <td>tom voted</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>tom wird aufhören</td>\n",
              "      <td>tomll will</td>\n",
              "      <td>tom will quit</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>zeig es uns</td>\n",
              "      <td>show it to</td>\n",
              "      <td>show it to us</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>du hast arbeit</td>\n",
              "      <td>you have a job</td>\n",
              "      <td>you have work to do</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9995</th>\n",
              "      <td>seid ihr nicht ärzte</td>\n",
              "      <td>arent you doctors</td>\n",
              "      <td>arent you doctors</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9996</th>\n",
              "      <td>seid ihr mutig</td>\n",
              "      <td>are you courageous</td>\n",
              "      <td>are you courageous</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9997</th>\n",
              "      <td>sind die für mich</td>\n",
              "      <td>are you my here</td>\n",
              "      <td>are those for me</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9998</th>\n",
              "      <td>sie hat eine lange nase</td>\n",
              "      <td>she works a  day</td>\n",
              "      <td>she has a long nose</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9999</th>\n",
              "      <td>ich will wasser</td>\n",
              "      <td>i want more</td>\n",
              "      <td>i want some water</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10000 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                   German Input                Predicted             Expected\n",
              "0     tom hat ein bier bestellt     tom asked a beer       tom ordered a beer\n",
              "1                    tom wählte        tom kneeled                  tom voted\n",
              "2             tom wird aufhören         tomll will              tom will quit\n",
              "3                   zeig es uns          show it to             show it to us\n",
              "4                du hast arbeit       you have a job      you have work to do\n",
              "...                         ...                      ...                  ...\n",
              "9995       seid ihr nicht ärzte   arent you doctors         arent you doctors\n",
              "9996             seid ihr mutig  are you courageous        are you courageous\n",
              "9997          sind die für mich      are you my here         are those for me\n",
              "9998    sie hat eine lange nase      she works a  day     she has a long nose\n",
              "9999            ich will wasser         i want more         i want some water\n",
              "\n",
              "[10000 rows x 3 columns]"
            ]
          },
          "execution_count": 67,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "pred_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 520
        },
        "id": "V9pcP8TnqYI2",
        "outputId": "238597ef-f1bb-435e-da9a-4dd84e5d9cac",
        "scrolled": true
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>German Input</th>\n",
              "      <th>Predicted</th>\n",
              "      <th>Expected</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3267</th>\n",
              "      <td>wer verkauft das</td>\n",
              "      <td>who says that</td>\n",
              "      <td>who sells this</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>410</th>\n",
              "      <td>wir wollen das wissen</td>\n",
              "      <td>we want to  why</td>\n",
              "      <td>we want to know</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6897</th>\n",
              "      <td>der hund bellte</td>\n",
              "      <td>the dog wants</td>\n",
              "      <td>the dog barked</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4902</th>\n",
              "      <td>tom ist ein starkes kind</td>\n",
              "      <td>tom is a clever boy</td>\n",
              "      <td>tom is a strong kid</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>133</th>\n",
              "      <td>tom sieht stark aus</td>\n",
              "      <td>tom looks naive</td>\n",
              "      <td>tom looks strong</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5971</th>\n",
              "      <td>nimm das</td>\n",
              "      <td>take this</td>\n",
              "      <td>grab that</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4336</th>\n",
              "      <td>darf ich deine hand halten</td>\n",
              "      <td>may i have your name</td>\n",
              "      <td>may i hold your hand</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5853</th>\n",
              "      <td>ich brauche meine brille</td>\n",
              "      <td>i need my shoes</td>\n",
              "      <td>i need my glasses</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>456</th>\n",
              "      <td>er fing zu weinen an</td>\n",
              "      <td>he began to cry</td>\n",
              "      <td>he began to cry</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7006</th>\n",
              "      <td>ruf meinen mann an</td>\n",
              "      <td>my  boy</td>\n",
              "      <td>call my husband</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8673</th>\n",
              "      <td>haben sie ein buch</td>\n",
              "      <td>do you have a book</td>\n",
              "      <td>do you have a book</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3882</th>\n",
              "      <td>tom scheint zu schlafen</td>\n",
              "      <td>tom seems quiet</td>\n",
              "      <td>tom looks asleep</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>469</th>\n",
              "      <td>das leben ist sehr schwer</td>\n",
              "      <td>its very  big</td>\n",
              "      <td>life is very hard</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5131</th>\n",
              "      <td>du bist der süßeste</td>\n",
              "      <td>youre the expert</td>\n",
              "      <td>youre the sweetest</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9468</th>\n",
              "      <td>ich werde die nicht unterschreiben</td>\n",
              "      <td>im not  this</td>\n",
              "      <td>i wont sign this</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                            German Input                Predicted  \\\n",
              "3267                    wer verkauft das       who says that        \n",
              "410                wir wollen das wissen       we want to  why      \n",
              "6897                     der hund bellte       the dog wants        \n",
              "4902            tom ist ein starkes kind   tom is a clever boy      \n",
              "133                  tom sieht stark aus     tom looks naive        \n",
              "5971                            nimm das          take this         \n",
              "4336          darf ich deine hand halten  may i have your name      \n",
              "5853            ich brauche meine brille      i need my shoes       \n",
              "456                 er fing zu weinen an      he began to cry       \n",
              "7006                  ruf meinen mann an             my  boy        \n",
              "8673                  haben sie ein buch    do you have a book      \n",
              "3882             tom scheint zu schlafen     tom seems quiet        \n",
              "469            das leben ist sehr schwer        its very  big       \n",
              "5131                 du bist der süßeste    youre the expert        \n",
              "9468  ich werde die nicht unterschreiben         im not  this       \n",
              "\n",
              "                  Expected  \n",
              "3267        who sells this  \n",
              "410        we want to know  \n",
              "6897        the dog barked  \n",
              "4902   tom is a strong kid  \n",
              "133       tom looks strong  \n",
              "5971             grab that  \n",
              "4336  may i hold your hand  \n",
              "5853     i need my glasses  \n",
              "456        he began to cry  \n",
              "7006       call my husband  \n",
              "8673    do you have a book  \n",
              "3882      tom looks asleep  \n",
              "469      life is very hard  \n",
              "5131    youre the sweetest  \n",
              "9468      i wont sign this  "
            ]
          },
          "execution_count": 68,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# print 15 rows randomly\n",
        "pred_df.sample(15)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 3. Combining both models\n"
      ],
      "metadata": {
        "id": "zJ80UPF4IHie"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n",
        "\n",
        "tokenizer_t = AutoTokenizer.from_pretrained(\"Helsinki-NLP/opus-mt-mul-en\")\n",
        "\n",
        "model_t = AutoModelForSeq2SeqLM.from_pretrained(\"Helsinki-NLP/opus-mt-mul-en\")"
      ],
      "metadata": {
        "id": "DxbEa6arABAX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "kannada_context = \"ಇಂದು ಒಳ್ಳೆಯ ದಿನ. ನಾನು ನದಿಯ ಮೂಲಕ ನಡೆಯಲು ಉತ್ತಮ ಸಮಯವನ್ನು ಹೊಂದಿದ್ದೇನೆ. ಹವಾಮಾನವು ಬಿಸಿಲು ಮತ್ತು ಬಿಸಿಯಾಗಿರುತ್ತದೆ.\"\n",
        "kannada_question = \"ಇಂದು ಹೇಗಿದೆ?\"\n"
      ],
      "metadata": {
        "id": "C3lhlNxMsgQ7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "batch = tokenizer_t([kannada_context], return_tensors=\"pt\")\n",
        "\n",
        "generated_ids = model_t.generate(**batch)\n",
        "eng_context = tokenizer_t.batch_decode(generated_ids, skip_special_tokens=True)[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9I8fLE_uvOQd",
        "outputId": "299d1f18-f8c3-40b1-b1a0-ffa0ece5279d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/generation_utils.py:1207: UserWarning: Neither `max_length` nor `max_new_tokens` have been set, `max_length` will default to 512 (`self.config.max_length`). Controlling `max_length` via the config is deprecated and `max_length` will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
            "  UserWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "eng_context"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "kacvdd4hvcNV",
        "outputId": "bec6fba8-8f09-4e70-c37b-bf94f1712439"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'It is a good day. I have a good time to walk through the river. The air is a spring and a spring.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "batch = tokenizer_t([kannada_question], return_tensors=\"pt\")\n",
        "\n",
        "generated_ids = model_t.generate(**batch)\n",
        "english_question = tokenizer_t.batch_decode(generated_ids, skip_special_tokens=True)[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "acq8Ccy1verE",
        "outputId": "d9008545-37ca-441f-cab3-6447727843f0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/generation_utils.py:1207: UserWarning: Neither `max_length` nor `max_new_tokens` have been set, `max_length` will default to 512 (`self.config.max_length`). Controlling `max_length` via the config is deprecated and `max_length` will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
            "  UserWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "english_question"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "bVcpex6dvtzD",
        "outputId": "97db2a06-1233-4496-a3af-d99ce5e030e0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'What about today?'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "question, text = english_question,eng_context"
      ],
      "metadata": {
        "id": "vcUzmfX6wXd-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "inputs = tokenizer(question, text, return_tensors=\"pt\")\n",
        "outputs = model(**inputs)"
      ],
      "metadata": {
        "id": "UT_aunGtwx2j"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "answer_start_index = outputs.start_logits.argmax()\n",
        "answer_end_index = outputs.end_logits.argmax()\n",
        "predict_answer_tokens = inputs.input_ids[0, answer_start_index : answer_end_index + 1]\n",
        "tokenizer.decode(predict_answer_tokens)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "_St12SPSw1t3",
        "outputId": "a20bfb48-f389-4adb-dc57-2380f77e0e06"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'It is a good day. I have a good time to walk through the river'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        ""
      ],
      "metadata": {
        "id": "ifgfym1zZaEK"
      }
    }
  ]
}