get_mhub_metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "API credentials have automatically been injected for your active subscriptions.  \n",
       "The following environment variables are now available:\n",
       "* `GEODB_API_SERVER_PORT`, `GEODB_API_SERVER_URL`, `GEODB_AUTH_AUD`, `GEODB_AUTH_CLIENT_ID`, `GEODB_AUTH_CLIENT_SECRET`, `GEODB_AUTH_DOMAIN`\n",
       "* `SH_CLIENT_ID`, `SH_CLIENT_NAME`, `SH_CLIENT_SECRET`, `SH_INSTANCE_ID`\n",
       "\n",
       "The following additional environment variables have been loaded from `~/custom.env`:\n",
       "* `AWS_BUCKET`\n"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import boto3\n",
    "import requests\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from edc import setup_environment_variables\n",
    "\n",
    "setup_environment_variables()\n",
    "bucket_name = %env AWS_BUCKET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>s3requests</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aade3d2484804fafa9bc075baa755513</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b703127c649e459ebb5fb70ea0000ddd</td>\n",
       "      <td>22411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>616500a3baed47c7bbcad3222658c5e5</td>\n",
       "      <td>3550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>98721703b76243ee9bd25509f23c24aa</td>\n",
       "      <td>293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>657742086071402a8c426e7fff9a3d6b</td>\n",
       "      <td>11938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1e19775eab6647418971b750e133529f</td>\n",
       "      <td>5837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>94dc3d7fcc654ee38b615971ed6fa6ff</td>\n",
       "      <td>51780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>10a6bbc2d49543f9a623542c74e6e050</td>\n",
       "      <td>41223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>6c8268fb4d454e269e3db22a9373434b</td>\n",
       "      <td>3558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1a59306b151b45ff92202a08280c9c25</td>\n",
       "      <td>4928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>9b778ab8a8b54976bcc7837a6014c864</td>\n",
       "      <td>12348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>58450d2e1f374e0d94519a22419aca20</td>\n",
       "      <td>5189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>473cdd2f407d44cf82af6a7e67039360</td>\n",
       "      <td>50726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>c1a70baf47e94d4d9498d5b0902264b2</td>\n",
       "      <td>88523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>631083feac344cb395a798ebd2439941</td>\n",
       "      <td>3733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1e28675732354c079675d195ac4968c7</td>\n",
       "      <td>81101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>8a0325c4d6e148578677390971b73ccc</td>\n",
       "      <td>52104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2b89da38ba604960901b0f4102738bb3</td>\n",
       "      <td>71202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>15831ca79f754b8f898e451cf1fc44dd</td>\n",
       "      <td>52756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>dbc4188882d24938a2debe6be29c1dc1</td>\n",
       "      <td>118367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>72646899d5434802b56b012eaec2dda0</td>\n",
       "      <td>25811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>4367d2fc43de495e98028bcd35db35ed</td>\n",
       "      <td>116058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>8065cc3890b64332972b3235a8bec24c</td>\n",
       "      <td>86003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>d5d2d591573744af8fac0ec3692e35dd</td>\n",
       "      <td>108680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>333c313e500944eab0b79401baa1ea83</td>\n",
       "      <td>81811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>5c0e0c98596f49d7a3567c07e3f00bb1</td>\n",
       "      <td>141021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>930d8cadf99746bcaa8959dc11cd73a2</td>\n",
       "      <td>26530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0527f273ce5e4cc0b7beff9dc6b26981</td>\n",
       "      <td>124656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>866579a6881c4e7aa588b679d4fed065</td>\n",
       "      <td>91126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>a6db3f73130841f787e9a3fce8d3e743</td>\n",
       "      <td>109492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>cb731d2767c940cabce5f7d4cd7c89e6</td>\n",
       "      <td>82733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>4ad0ccbb9ad04a6e8671e24cfefe13ca</td>\n",
       "      <td>171181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>fb1e013d6272426795672aead7ddf8b6</td>\n",
       "      <td>26376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>6bbc18628f464f5ba43cf7aca6c3e5ed</td>\n",
       "      <td>154460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>2fafa23b0bde4b78ab948922304a35b1</td>\n",
       "      <td>90971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>6310ecc6b254462a80f4eaf7e4829bf0</td>\n",
       "      <td>122373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>d3522b5ab0644ebe94b781d4e79660b4</td>\n",
       "      <td>83489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>0a312ca669d94531b27a5d3d6265cba8</td>\n",
       "      <td>237824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>b893c79df427434f92044772529f9492</td>\n",
       "      <td>64810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>778d0073031e4248b4af1b6d6041e5b1</td>\n",
       "      <td>210466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>2eaf7b8256b44e3a976905389b01a71e</td>\n",
       "      <td>91796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>fb37642117e94830a3a3c0988faf7594</td>\n",
       "      <td>173168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>534771466b164775ac8c286fc30e1f1f</td>\n",
       "      <td>123012</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  id  s3requests\n",
       "0   aade3d2484804fafa9bc075baa755513          69\n",
       "1   b703127c649e459ebb5fb70ea0000ddd       22411\n",
       "2   616500a3baed47c7bbcad3222658c5e5        3550\n",
       "3   98721703b76243ee9bd25509f23c24aa         293\n",
       "4   657742086071402a8c426e7fff9a3d6b       11938\n",
       "5   1e19775eab6647418971b750e133529f        5837\n",
       "6   94dc3d7fcc654ee38b615971ed6fa6ff       51780\n",
       "7   10a6bbc2d49543f9a623542c74e6e050       41223\n",
       "8   6c8268fb4d454e269e3db22a9373434b        3558\n",
       "9   1a59306b151b45ff92202a08280c9c25        4928\n",
       "10  9b778ab8a8b54976bcc7837a6014c864       12348\n",
       "11  58450d2e1f374e0d94519a22419aca20        5189\n",
       "12  473cdd2f407d44cf82af6a7e67039360       50726\n",
       "13  c1a70baf47e94d4d9498d5b0902264b2       88523\n",
       "14  631083feac344cb395a798ebd2439941        3733\n",
       "15  1e28675732354c079675d195ac4968c7       81101\n",
       "16  8a0325c4d6e148578677390971b73ccc       52104\n",
       "17  2b89da38ba604960901b0f4102738bb3       71202\n",
       "18  15831ca79f754b8f898e451cf1fc44dd       52756\n",
       "19  dbc4188882d24938a2debe6be29c1dc1      118367\n",
       "20  72646899d5434802b56b012eaec2dda0       25811\n",
       "21  4367d2fc43de495e98028bcd35db35ed      116058\n",
       "22  8065cc3890b64332972b3235a8bec24c       86003\n",
       "23  d5d2d591573744af8fac0ec3692e35dd      108680\n",
       "24  333c313e500944eab0b79401baa1ea83       81811\n",
       "25  5c0e0c98596f49d7a3567c07e3f00bb1      141021\n",
       "26  930d8cadf99746bcaa8959dc11cd73a2       26530\n",
       "27  0527f273ce5e4cc0b7beff9dc6b26981      124656\n",
       "28  866579a6881c4e7aa588b679d4fed065       91126\n",
       "29  a6db3f73130841f787e9a3fce8d3e743      109492\n",
       "30  cb731d2767c940cabce5f7d4cd7c89e6       82733\n",
       "31  4ad0ccbb9ad04a6e8671e24cfefe13ca      171181\n",
       "32  fb1e013d6272426795672aead7ddf8b6       26376\n",
       "33  6bbc18628f464f5ba43cf7aca6c3e5ed      154460\n",
       "34  2fafa23b0bde4b78ab948922304a35b1       90971\n",
       "35  6310ecc6b254462a80f4eaf7e4829bf0      122373\n",
       "36  d3522b5ab0644ebe94b781d4e79660b4       83489\n",
       "37  0a312ca669d94531b27a5d3d6265cba8      237824\n",
       "38  b893c79df427434f92044772529f9492       64810\n",
       "39  778d0073031e4248b4af1b6d6041e5b1      210466\n",
       "40  2eaf7b8256b44e3a976905389b01a71e       91796\n",
       "41  fb37642117e94830a3a3c0988faf7594      173168\n",
       "42  534771466b164775ac8c286fc30e1f1f      123012"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('requests.txt') as log:\n",
    "    loglines = [line[line.rfind(']')+2:].rstrip() for line in log]\n",
    "    s3requests = []\n",
    "    \n",
    "    for idx,ll in enumerate(loglines):\n",
    "        if ll.startswith('g') and ll != loglines[-1] and loglines[idx+1].startswith('N'):\n",
    "            s3requests.append([ll.split(' ')[-1],\n",
    "                   int(loglines[idx+1].split(':')[-1])])# + int(loglines[idx+2].split(':')[-1])])\n",
    "            \n",
    "s3requests = pd.DataFrame(s3requests, columns=['id', 's3requests'])\n",
    "s3requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Status of unfinished tiles []\n",
      "                                  id      runtime                       path  \\\n",
      "0   b703127c649e459ebb5fb70ea0000ddd   655.667576  mapchete_cubes/mdc_02_01/   \n",
      "1   616500a3baed47c7bbcad3222658c5e5   464.066579  mapchete_cubes/mdc_02_01/   \n",
      "2   98721703b76243ee9bd25509f23c24aa   784.729690  mapchete_cubes/mdc_02_01/   \n",
      "3   657742086071402a8c426e7fff9a3d6b   629.219358  mapchete_cubes/mdc_02_01/   \n",
      "4   1e19775eab6647418971b750e133529f   601.767835  mapchete_cubes/mdc_02_01/   \n",
      "5   94dc3d7fcc654ee38b615971ed6fa6ff   573.001364  mapchete_cubes/mdc_02_01/   \n",
      "6   10a6bbc2d49543f9a623542c74e6e050  1129.352243  mapchete_cubes/mdc_04_01/   \n",
      "7   6c8268fb4d454e269e3db22a9373434b   749.995984  mapchete_cubes/mdc_04_01/   \n",
      "8   1a59306b151b45ff92202a08280c9c25  1397.318961  mapchete_cubes/mdc_04_01/   \n",
      "9   9b778ab8a8b54976bcc7837a6014c864  1076.571560  mapchete_cubes/mdc_04_01/   \n",
      "10  58450d2e1f374e0d94519a22419aca20  1041.177847  mapchete_cubes/mdc_04_01/   \n",
      "11  473cdd2f407d44cf82af6a7e67039360  1104.777646  mapchete_cubes/mdc_04_01/   \n",
      "12  c1a70baf47e94d4d9498d5b0902264b2  1621.013915  mapchete_cubes/mdc_06_01/   \n",
      "13  631083feac344cb395a798ebd2439941  1041.759525  mapchete_cubes/mdc_06_01/   \n",
      "14  1e28675732354c079675d195ac4968c7  1828.350021  mapchete_cubes/mdc_06_01/   \n",
      "15  8a0325c4d6e148578677390971b73ccc  1488.983791  mapchete_cubes/mdc_06_01/   \n",
      "16  2b89da38ba604960901b0f4102738bb3  1387.147216  mapchete_cubes/mdc_06_01/   \n",
      "17  15831ca79f754b8f898e451cf1fc44dd  1450.155521  mapchete_cubes/mdc_06_01/   \n",
      "18  dbc4188882d24938a2debe6be29c1dc1  2178.807789  mapchete_cubes/mdc_08_01/   \n",
      "19  72646899d5434802b56b012eaec2dda0  1330.166794  mapchete_cubes/mdc_08_01/   \n",
      "20  4367d2fc43de495e98028bcd35db35ed  2338.528545  mapchete_cubes/mdc_08_01/   \n",
      "21  8065cc3890b64332972b3235a8bec24c  1806.762923  mapchete_cubes/mdc_08_01/   \n",
      "22  d5d2d591573744af8fac0ec3692e35dd  1808.941426  mapchete_cubes/mdc_08_01/   \n",
      "23  333c313e500944eab0b79401baa1ea83  1764.039313  mapchete_cubes/mdc_08_01/   \n",
      "24  5c0e0c98596f49d7a3567c07e3f00bb1  2719.420224  mapchete_cubes/mdc_10_01/   \n",
      "25  930d8cadf99746bcaa8959dc11cd73a2  1695.721057  mapchete_cubes/mdc_10_01/   \n",
      "26  0527f273ce5e4cc0b7beff9dc6b26981  2947.752656  mapchete_cubes/mdc_10_01/   \n",
      "27  866579a6881c4e7aa588b679d4fed065  2292.301462  mapchete_cubes/mdc_10_01/   \n",
      "28  a6db3f73130841f787e9a3fce8d3e743  2349.025240  mapchete_cubes/mdc_10_01/   \n",
      "29  cb731d2767c940cabce5f7d4cd7c89e6  2334.730431  mapchete_cubes/mdc_10_01/   \n",
      "30  4ad0ccbb9ad04a6e8671e24cfefe13ca  3220.709962  mapchete_cubes/mdc_12_01/   \n",
      "31  fb1e013d6272426795672aead7ddf8b6  2002.482632  mapchete_cubes/mdc_12_01/   \n",
      "32  6bbc18628f464f5ba43cf7aca6c3e5ed  3539.187790  mapchete_cubes/mdc_12_01/   \n",
      "33  2fafa23b0bde4b78ab948922304a35b1  2803.481610  mapchete_cubes/mdc_12_01/   \n",
      "34  6310ecc6b254462a80f4eaf7e4829bf0  2786.607670  mapchete_cubes/mdc_12_01/   \n",
      "35  d3522b5ab0644ebe94b781d4e79660b4  2801.492855  mapchete_cubes/mdc_12_01/   \n",
      "36  0a312ca669d94531b27a5d3d6265cba8  4567.913563  mapchete_cubes/mdc_17_01/   \n",
      "37  b893c79df427434f92044772529f9492  2876.555089  mapchete_cubes/mdc_17_01/   \n",
      "38  778d0073031e4248b4af1b6d6041e5b1  5041.536348  mapchete_cubes/mdc_17_01/   \n",
      "39  2eaf7b8256b44e3a976905389b01a71e  3893.084784  mapchete_cubes/mdc_17_01/   \n",
      "40  fb37642117e94830a3a3c0988faf7594  3914.917590  mapchete_cubes/mdc_17_01/   \n",
      "41  534771466b164775ac8c286fc30e1f1f  3839.458825  mapchete_cubes/mdc_17_01/   \n",
      "\n",
      "    s3requests  \n",
      "0        22411  \n",
      "1         3550  \n",
      "2          293  \n",
      "3        11938  \n",
      "4         5837  \n",
      "5        51780  \n",
      "6        41223  \n",
      "7         3558  \n",
      "8         4928  \n",
      "9        12348  \n",
      "10        5189  \n",
      "11       50726  \n",
      "12       88523  \n",
      "13        3733  \n",
      "14       81101  \n",
      "15       52104  \n",
      "16       71202  \n",
      "17       52756  \n",
      "18      118367  \n",
      "19       25811  \n",
      "20      116058  \n",
      "21       86003  \n",
      "22      108680  \n",
      "23       81811  \n",
      "24      141021  \n",
      "25       26530  \n",
      "26      124656  \n",
      "27       91126  \n",
      "28      109492  \n",
      "29       82733  \n",
      "30      171181  \n",
      "31       26376  \n",
      "32      154460  \n",
      "33       90971  \n",
      "34      122373  \n",
      "35       83489  \n",
      "36      237824  \n",
      "37       64810  \n",
      "38      210466  \n",
      "39       91796  \n",
      "40      173168  \n",
      "41      123012  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th colspan=\"3\" halign=\"left\">runtime</th>\n",
       "      <th>s3requests</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>list</th>\n",
       "      <th>sum</th>\n",
       "      <th>amin</th>\n",
       "      <th>amax</th>\n",
       "      <th>sum</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>path</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mapchete_cubes/mdc_02_01/</th>\n",
       "      <td>[b703127c649e459ebb5fb70ea0000ddd, 616500a3bae...</td>\n",
       "      <td>3708.452402</td>\n",
       "      <td>464.066579</td>\n",
       "      <td>784.729690</td>\n",
       "      <td>95809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mapchete_cubes/mdc_04_01/</th>\n",
       "      <td>[10a6bbc2d49543f9a623542c74e6e050, 6c8268fb4d4...</td>\n",
       "      <td>6499.194240</td>\n",
       "      <td>749.995984</td>\n",
       "      <td>1397.318961</td>\n",
       "      <td>117972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mapchete_cubes/mdc_06_01/</th>\n",
       "      <td>[c1a70baf47e94d4d9498d5b0902264b2, 631083feac3...</td>\n",
       "      <td>8817.409989</td>\n",
       "      <td>1041.759525</td>\n",
       "      <td>1828.350021</td>\n",
       "      <td>349419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mapchete_cubes/mdc_08_01/</th>\n",
       "      <td>[dbc4188882d24938a2debe6be29c1dc1, 72646899d54...</td>\n",
       "      <td>11227.246790</td>\n",
       "      <td>1330.166794</td>\n",
       "      <td>2338.528545</td>\n",
       "      <td>536730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mapchete_cubes/mdc_10_01/</th>\n",
       "      <td>[5c0e0c98596f49d7a3567c07e3f00bb1, 930d8cadf99...</td>\n",
       "      <td>14338.951070</td>\n",
       "      <td>1695.721057</td>\n",
       "      <td>2947.752656</td>\n",
       "      <td>575558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mapchete_cubes/mdc_12_01/</th>\n",
       "      <td>[4ad0ccbb9ad04a6e8671e24cfefe13ca, fb1e013d627...</td>\n",
       "      <td>17153.962519</td>\n",
       "      <td>2002.482632</td>\n",
       "      <td>3539.187790</td>\n",
       "      <td>648850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mapchete_cubes/mdc_17_01/</th>\n",
       "      <td>[0a312ca669d94531b27a5d3d6265cba8, b893c79df42...</td>\n",
       "      <td>24133.466200</td>\n",
       "      <td>2876.555089</td>\n",
       "      <td>5041.536348</td>\n",
       "      <td>901076</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                          id  \\\n",
       "                                                                        list   \n",
       "path                                                                           \n",
       "mapchete_cubes/mdc_02_01/  [b703127c649e459ebb5fb70ea0000ddd, 616500a3bae...   \n",
       "mapchete_cubes/mdc_04_01/  [10a6bbc2d49543f9a623542c74e6e050, 6c8268fb4d4...   \n",
       "mapchete_cubes/mdc_06_01/  [c1a70baf47e94d4d9498d5b0902264b2, 631083feac3...   \n",
       "mapchete_cubes/mdc_08_01/  [dbc4188882d24938a2debe6be29c1dc1, 72646899d54...   \n",
       "mapchete_cubes/mdc_10_01/  [5c0e0c98596f49d7a3567c07e3f00bb1, 930d8cadf99...   \n",
       "mapchete_cubes/mdc_12_01/  [4ad0ccbb9ad04a6e8671e24cfefe13ca, fb1e013d627...   \n",
       "mapchete_cubes/mdc_17_01/  [0a312ca669d94531b27a5d3d6265cba8, b893c79df42...   \n",
       "\n",
       "                                runtime                           s3requests  \n",
       "                                    sum         amin         amax        sum  \n",
       "path                                                                          \n",
       "mapchete_cubes/mdc_02_01/   3708.452402   464.066579   784.729690      95809  \n",
       "mapchete_cubes/mdc_04_01/   6499.194240   749.995984  1397.318961     117972  \n",
       "mapchete_cubes/mdc_06_01/   8817.409989  1041.759525  1828.350021     349419  \n",
       "mapchete_cubes/mdc_08_01/  11227.246790  1330.166794  2338.528545     536730  \n",
       "mapchete_cubes/mdc_10_01/  14338.951070  1695.721057  2947.752656     575558  \n",
       "mapchete_cubes/mdc_12_01/  17153.962519  2002.482632  3539.187790     648850  \n",
       "mapchete_cubes/mdc_17_01/  24133.466200  2876.555089  5041.536348     901076  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobs = requests.get('https://demo-m.hub.eox.at/jobs/').json()\n",
    "jobs = [j for j in jobs if 'mdc' in j['properties']['config']['mapchete_config']['output']['path']]\n",
    "\n",
    "print('Status of unfinished tiles', [j['properties']['progress_data'] for j in jobs if j['properties']['state'] != 'SUCCESS'])\n",
    "\n",
    "results = pd.DataFrame([{\n",
    "    'id': j['id'],\n",
    "    'runtime': j['properties']['runtime'],\n",
    "    'path': j['properties']['config']['mapchete_config']['output']['path'].replace('s3://eox-masterdatacube/', '')\n",
    "} for j in jobs])\n",
    "\n",
    "results = pd.merge(results,s3requests, how='left')\n",
    "\n",
    "print(results)\n",
    "\n",
    "metrics = pd.DataFrame(results.groupby('path').agg({'id': list,\n",
    "                                                    'runtime': [np.sum, np.min, np.max],\n",
    "                                                    's3requests': np.sum}))\n",
    "\n",
    "metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th colspan=\"3\" halign=\"left\">runtime</th>\n",
       "      <th>s3requests</th>\n",
       "      <th>size</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>list</th>\n",
       "      <th>sum</th>\n",
       "      <th>amin</th>\n",
       "      <th>amax</th>\n",
       "      <th>sum</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>path</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mapchete_cubes/mdc_02_01/</th>\n",
       "      <td>[b703127c649e459ebb5fb70ea0000ddd, 616500a3bae...</td>\n",
       "      <td>3708.452402</td>\n",
       "      <td>464.066579</td>\n",
       "      <td>784.729690</td>\n",
       "      <td>95809</td>\n",
       "      <td>7.212899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mapchete_cubes/mdc_04_01/</th>\n",
       "      <td>[10a6bbc2d49543f9a623542c74e6e050, 6c8268fb4d4...</td>\n",
       "      <td>6499.194240</td>\n",
       "      <td>749.995984</td>\n",
       "      <td>1397.318961</td>\n",
       "      <td>117972</td>\n",
       "      <td>13.393613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mapchete_cubes/mdc_06_01/</th>\n",
       "      <td>[c1a70baf47e94d4d9498d5b0902264b2, 631083feac3...</td>\n",
       "      <td>8817.409989</td>\n",
       "      <td>1041.759525</td>\n",
       "      <td>1828.350021</td>\n",
       "      <td>349419</td>\n",
       "      <td>18.780106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mapchete_cubes/mdc_08_01/</th>\n",
       "      <td>[dbc4188882d24938a2debe6be29c1dc1, 72646899d54...</td>\n",
       "      <td>11227.246790</td>\n",
       "      <td>1330.166794</td>\n",
       "      <td>2338.528545</td>\n",
       "      <td>536730</td>\n",
       "      <td>23.463265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mapchete_cubes/mdc_10_01/</th>\n",
       "      <td>[5c0e0c98596f49d7a3567c07e3f00bb1, 930d8cadf99...</td>\n",
       "      <td>14338.951070</td>\n",
       "      <td>1695.721057</td>\n",
       "      <td>2947.752656</td>\n",
       "      <td>575558</td>\n",
       "      <td>29.429667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mapchete_cubes/mdc_12_01/</th>\n",
       "      <td>[4ad0ccbb9ad04a6e8671e24cfefe13ca, fb1e013d627...</td>\n",
       "      <td>17153.962519</td>\n",
       "      <td>2002.482632</td>\n",
       "      <td>3539.187790</td>\n",
       "      <td>648850</td>\n",
       "      <td>34.245206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mapchete_cubes/mdc_17_01/</th>\n",
       "      <td>[0a312ca669d94531b27a5d3d6265cba8, b893c79df42...</td>\n",
       "      <td>24133.466200</td>\n",
       "      <td>2876.555089</td>\n",
       "      <td>5041.536348</td>\n",
       "      <td>901076</td>\n",
       "      <td>48.370471</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                          id  \\\n",
       "                                                                        list   \n",
       "path                                                                           \n",
       "mapchete_cubes/mdc_02_01/  [b703127c649e459ebb5fb70ea0000ddd, 616500a3bae...   \n",
       "mapchete_cubes/mdc_04_01/  [10a6bbc2d49543f9a623542c74e6e050, 6c8268fb4d4...   \n",
       "mapchete_cubes/mdc_06_01/  [c1a70baf47e94d4d9498d5b0902264b2, 631083feac3...   \n",
       "mapchete_cubes/mdc_08_01/  [dbc4188882d24938a2debe6be29c1dc1, 72646899d54...   \n",
       "mapchete_cubes/mdc_10_01/  [5c0e0c98596f49d7a3567c07e3f00bb1, 930d8cadf99...   \n",
       "mapchete_cubes/mdc_12_01/  [4ad0ccbb9ad04a6e8671e24cfefe13ca, fb1e013d627...   \n",
       "mapchete_cubes/mdc_17_01/  [0a312ca669d94531b27a5d3d6265cba8, b893c79df42...   \n",
       "\n",
       "                                runtime                           s3requests  \\\n",
       "                                    sum         amin         amax        sum   \n",
       "path                                                                           \n",
       "mapchete_cubes/mdc_02_01/   3708.452402   464.066579   784.729690      95809   \n",
       "mapchete_cubes/mdc_04_01/   6499.194240   749.995984  1397.318961     117972   \n",
       "mapchete_cubes/mdc_06_01/   8817.409989  1041.759525  1828.350021     349419   \n",
       "mapchete_cubes/mdc_08_01/  11227.246790  1330.166794  2338.528545     536730   \n",
       "mapchete_cubes/mdc_10_01/  14338.951070  1695.721057  2947.752656     575558   \n",
       "mapchete_cubes/mdc_12_01/  17153.962519  2002.482632  3539.187790     648850   \n",
       "mapchete_cubes/mdc_17_01/  24133.466200  2876.555089  5041.536348     901076   \n",
       "\n",
       "                                size  \n",
       "                                      \n",
       "path                                  \n",
       "mapchete_cubes/mdc_02_01/   7.212899  \n",
       "mapchete_cubes/mdc_04_01/  13.393613  \n",
       "mapchete_cubes/mdc_06_01/  18.780106  \n",
       "mapchete_cubes/mdc_08_01/  23.463265  \n",
       "mapchete_cubes/mdc_10_01/  29.429667  \n",
       "mapchete_cubes/mdc_12_01/  34.245206  \n",
       "mapchete_cubes/mdc_17_01/  48.370471  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s3 = boto3.resource('s3')\n",
    "bk = s3.Bucket(bucket_name)\n",
    "\n",
    "sizes = []\n",
    "\n",
    "for path in metrics.index:\n",
    "    objects = bk.objects.filter(Prefix=path+'5')\n",
    "    sizes.append(sum([o.size for o in objects])/2**30)\n",
    "    \n",
    "metrics['size'] = sizes\n",
    "metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics.to_csv('metrics.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "EDC 0.20.1 (Python3)",
   "language": "python",
   "name": "edc"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}