diff --git a/notebooks/proba_functions.ipynb b/notebooks/proba_functions.ipynb
index 92ca1ec..ee35a2b 100644
--- a/notebooks/proba_functions.ipynb
+++ b/notebooks/proba_functions.ipynb
@@ -1,637 +1,2641 @@
 {
  "cells": [
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Make distribution tables to calculate probabilities of transfer\n",
+    "## Compute probability of missing a transfer from delays distributions\n",
     "\n",
-    "<div class='alert alert-info'><b>Any application without a proper name would be promptly killed.</b></div>"
+    "Let's first have a look at a slice of the dictionnary of distribution"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "Current session configs: <tt>{'conf': {'spark.app.name': 'lgptguys_final'}, 'kind': 'pyspark'}</tt><br>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<table>\n",
-       "<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>6849</td><td>application_1589299642358_1346</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1346/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster066.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1346_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>6852</td><td>application_1589299642358_1349</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1349/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster069.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1349_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>6858</td><td>application_1589299642358_1352</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1352/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster065.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1352_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>6861</td><td>application_1589299642358_1355</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1355/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster066.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1355_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>6866</td><td>application_1589299642358_1360</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1360/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster068.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1360_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>6867</td><td>application_1589299642358_1361</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1361/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster065.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1361_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>6869</td><td>application_1589299642358_1363</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1363/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster065.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1363_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>6871</td><td>application_1589299642358_1365</td><td>pyspark</td><td>busy</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1365/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster065.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1365_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>6872</td><td>application_1589299642358_1366</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1366/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster070.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1366_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>6875</td><td>application_1589299642358_1369</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1369/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster067.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1369_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>6876</td><td>application_1589299642358_1370</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1370/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster070.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1370_01_000001/ebouille\">Link</a></td><td></td></tr></table>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
+   "source": [
+    "import pickle \n",
+    "import gzip\n",
+    "from itertools import islice\n",
+    "import matplotlib as mlt \n",
+    "import matplotlib.pyplot as plt\n",
+    "import numpy as np \n",
+    "import pandas as pd \n",
+    "import math"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
    "source": [
-    "%%configure\n",
-    "{\"conf\": {\n",
-    "    \"spark.app.name\": \"lgptguys_final\"\n",
-    "}}"
+    "# Functon to take a slice from a dictionnary - head equivalent\n",
+    "def take(n, iterable):\n",
+    "    \"Return first n items of the iterable as a list\"\n",
+    "    return list(islice(iterable, n))"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Start Spark"
+    "Load dictionnaries of distributions"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Starting Spark application\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<table>\n",
-       "<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>6877</td><td>application_1589299642358_1371</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1371/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster071.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1371_01_000001/ebouille\">Link</a></td><td>✔</td></tr></table>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "SparkSession available as 'spark'.\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "An error was encountered:\n",
-      "unknown magic command '%spark'\n",
-      "UnknownMagic: unknown magic command '%spark'\n",
-      "\n"
+      "len dict_real :  12309\n",
+      "[('10.TA.1-11-B-j19-1.1.R__8590314', array([0, 2, 2, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0])), ('10.TA.1-11-B-j19-1.1.R__8590317', array([0, 3, 2, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0])), ('10.TA.1-11-B-j19-1.1.R__8594304', array([0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0])), ('10.TA.1-11-B-j19-1.1.R__8594307', array([0, 1, 5, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,\n",
+      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0])), ('10.TA.1-11-B-j19-1.1.R__8594310', array([0, 1, 3, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0]))]\n",
+      "len dict_all :  246968\n",
+      "[('1286.TA.26-32-j19-1.12.H__8591182', array([   0, 1158,  306,  162,   94,   24,   28,   21,    3,    2,    0,\n",
+      "          1,    0,    0,    0,    0,    1,    0,    0,    0,    0,    0,\n",
+      "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0])), ('1286.TA.26-32-j19-1.12.H__8591184', array([  1, 762, 552, 292, 118,  48,  13,   8,   0,   1,   1,   0,   0,\n",
+      "         0,   0,   0,   0,   0,   0,   0,   0,   1,   0,   0,   0,   0,\n",
+      "         0,   0,   0,   0,   0,   0])), ('1286.TA.26-32-j19-1.12.H__8591195', array([   0, 1083,  444,  143,   64,   35,   16,    9,    3,    1,    0,\n",
+      "          0,    0,    0,    0,    1,    0,    0,    0,    0,    1,    0,\n",
+      "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0])), ('1286.TA.26-32-j19-1.12.H__8591200', array([  2, 239, 227, 228, 212, 128,  74,  42,  29,  17,   3,   3,   2,\n",
+      "         0,   0,   0,   0,   0,   0,   0,   1,   0,   0,   0,   0,   0,\n",
+      "         0,   0,   0,   0,   0,   1])), ('1286.TA.26-32-j19-1.12.H__8591209', array([   0, 1151,  308,  169,   94,   24,   29,   16,    4,    3,    1,\n",
+      "          0,    0,    0,    0,    0,    1,    0,    0,    0,    0,    0,\n",
+      "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0]))]\n"
      ]
     }
    ],
    "source": [
-    "# Initialization\n",
-    "%%spark"
+    "with gzip.open(\"../data/distributions_geschaetzAndReal.pkl.gz\", \"rb\") as input_file:\n",
+    "    d_real = pickle.load(input_file)\n",
+    "\n",
+    "with gzip.open(\"../data/distributions.pickle\", \"rb\") as input_file:\n",
+    "    d_all = pickle.load(input_file)\n",
+    "\n",
+    "# display a slice of it\n",
+    "print('len dict_real : ', len(d_real))\n",
+    "print(take(5, d_real.items()))\n",
+    "\n",
+    "# display a slice of it\n",
+    "print('len dict_all : ', len(d_all))\n",
+    "print(take(5, d_all.items()))"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Compute probability of missing a transfer from delays distributions\n",
+    "### Probability using cumulative distribution based on frequency of delays \n",
     "\n",
-    "Let's first have a look at a slice of the dictionnary of distribution"
+    "When we have __enough data__ and no ambiguity about `trip_id` and `stop_id` for a given distribution, then we can compute the probability $P(x \\leq X)$ for every x (delay in minute). \n",
+    "\n",
+    "Let's take a __threshold of 100__ sample points (=number of time we could measure a delay) as a minimum number of points to use this approach. \n",
+    "\n",
+    "_How many keys in our distionnary of distribution have at least this number of samples ?_"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
-    "import pickle \n",
-    "import gzip\n",
-    "from itertools import islice\n",
-    "import matplotlib as mlt \n",
-    "import matplotlib.pyplot as plt\n",
-    "import numpy as np \n",
-    "import pandas as pd "
+    "def plot_data_points_hist(dico):\n",
+    "    list_tot_points = []\n",
+    "    for key in dico:\n",
+    "        distrib = dico[key]\n",
+    "        list_tot_points.append(np.sum(distrib))\n",
+    "\n",
+    "    tot_per_key = np.array(list_tot_points)\n",
+    "    binwidth = 100\n",
+    "    n_keys_less_than_binwidth = np.sum(np.array(tot_per_key < binwidth))\n",
+    "    perc_key_to_recover = round(100 * ( n_keys_less_than_binwidth / len(tot_per_key) ), 2)\n",
+    "    plt.figure(figsize = (10,5))\n",
+    "    plt.hist(tot_per_key, bins = range(min(tot_per_key), max(tot_per_key) + binwidth, binwidth))\n",
+    "    plt.title(\"Total number of data points per trip_id / stop_id key. N keys with less than {0} points: {1} ({2}%)\"\\\n",
+    "             .format(binwidth, n_keys_less_than_binwidth, perc_key_to_recover))\n",
+    "    plt.xlabel('n data points')\n",
+    "    plt.ylabel('n keys')\n",
+    "    return plt.show()"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 5,
    "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Functon to take a slice from a dictionnary - head equivalent\n",
-    "def take(n, iterable):\n",
-    "    \"Return first n items of the iterable as a list\"\n",
-    "    return list(islice(iterable, n))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
    "outputs": [
     {
      "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAo4AAAFNCAYAAACOmu5nAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZglVXn48e/LjmwDMvKTGWRQiQZ3IYJxI2JgABFUUBRlCRENLpgYFYwRQVDQKGIAEQXZVEBiFBWDuIAryODCKjqyyM7ADPsO7++Pcy7cufTtPj3Tt5eZ7+d5+um6p7ZTVaeq3jp1qioyE0mSJGkky0x0BiRJkjQ1GDhKkiSpiYGjJEmSmhg4SpIkqYmBoyRJkpoYOEqSJKnJEh84RkRGxDMnQT7OiYh/nqB5rxwR342IOyLimw3Dbx4R141H3hZHROwSET+c6HyMl4j4QUTsNqhpRMSsur8stzjzmCgRcXdEPL1Pv90j4hd9+k3q5R4u75NFRDytrv9lhxmm+VgcER+PiJPHLoejMxXW+Xha2o61iyIitoqIb090PhZHRLw3Ig4dabgJCxzrQabz92hE3Nf1e5c+40yJgGYS2hFYB3hyZu40lhOeyANsZn4tM7dsGXYynwhaT5KZuXVmnrA481rcaUTEuqPdByPi+Ig4aFHn2SozV83MKwc9n8XRFaSe2ZN+ckR8fIKytdgy8691/T8CE3uhPFoTfeEQEW+KiF9FxL0Rcc4Q/V8YERfW/hdGxAu7+kVEHBoRt9W/QyMixjqPk+FYGxHH9V581P3mxoi4MyL+1K/MRcTH6riv6Ur7dERcW8e9JiI+0jPOdhFxSY1JfhURG42QxYOBQ7rG/0REXBwRD/fu2xHxkZ4Y6L4aB63dJ/9X98RIP+zqt2JEHBYRN0TEgog4KiKW7+r/+Zr+64iY2ZX+1oj4Qs+svgzsEhFPGW5BJyxwrAeZVTNzVeCvwHZdaV+bqHxNdvVAMdrttj7wp8x8eBB5UrtFOTkt4jYflG2A/5voTCwBNo2Iv5/oTGhSmA98nq6goyMiVgC+A5wMrAmcAHynpgPsBewAvAB4PrAd8M5xyPO4ioiXA88YotengFmZuTrwOuCgiNi4Z9xnADsBN/aMeyzw7Dru31MCpjfUcTYEvga8C5gGfBc4o9/xOyL+DlgjM8/rSp4LfAj4fu/wmfnJnhjoUOCczLx1mNXQHSN1B/H7ApsAzwX+Bngx8NGar5cAGwP/D/hFHZaIWAP4YGe4rnzdD/wA2HWYfEBmTvgfcDXwmtq9ImUnuqH+fb6mrQLcBzwK3F3/1gVeAvwauJ1SMI4AVuiadgLP7DPfc4BPAL8E7gJ+CKxd+20OXDdMPj8OfJOyQ98FXFw32n7ALcC1wJY98/oU8BvgTsrBYK2u/psBv6rL8Qdg855xD675vG+o5QH+tg53O3Ap8LqafgDwIPBQXWd7DjHuysDxwALgMkqBuq6r/77AX+pyXga8vmue9wOP1GnfXtO3BX5Xl/Na4OPDbPvNgeuAjwC31nW8S1f/NYATgXnANZSCvkzttzvwi55t/S7gz3U9HAnEMPncpi7PXcD1wL/3yePudd0fAdwB/BHYoiePx1LK3/XAQcCyPeMeBtwGHNQz7dk92+cP/bZ5TfvnljwNs767p7Es8F91vV8JvLuuw+WGGf9bwBuGSI+6jLfU7X4x5UC2V122B+vyfXe48lr7HQ8cDZxdt825wPoNy/bYvg48GTij5uU3lP38F33Gm9W93MAbKeXwuZSL6075vw04jbrfUk4I7+2Z1kXU/WOEeX0Y+GlX+sn02U94Yjn/DOUksEa/sgesQAlIntc13lOAe4HpwNrA9+r6nw/8nLpf9cz7AOC/a/fywD3AZ7qOG/cDa3WvQ0q5faT2uxs4Yrj9s88yfxw4ufH4uDul/N4FXEU9flD2mXMp+8etwKl95vXXmrfOeeWlnXVO2T8W1Olu3TXOHsDldZ5XAu8c4pj2Acr+cCOwR0P5/WdK8NCdtmXdrtGT39m1+1fAXl399gTOm6rH2j75Xo5yPnk+w5/Pn1XX9Zt60v+vzv9q6vl7iHFnUI5ZH6q/3wN8v6v/MpTj8JDHWOBjwFf69Ou7b9f+UcvQbsMMM1ze5wA7df1+K3Bt7X4z8KnaPRs4s3YfAby1z/R2oevYNOQwrRtvkH8sHJAdCJxHOchNrzvGJ7oLfs+4G1MOKstRDl6XA+/vKeDDBY5/oQR8K9ffhwwzr+58frzuIFvVeZ9IObj8B+UA+w7gqp55XU85Ga0C/A/1wFgL7W21cC8D/GP9Pb1r3L8Cz6nzWr4nX8tTrm4+QjlhvJqygz6rK68nD7P+D6GcONYC1gMuYeHAcSdKkL5MLYj3AE+t/Xan54Rc193z6vDPB24Gdugz782Bh4HPUS4QXlWn38n7iZQge7W6ff9EDX5751239fcoV4hPoxwAZw+TzxuBV9TuNYEX98nj7jWP/1rX9ZspJ6NOAPG/wJfqdn0KJVB5Z8+4763bbuUhpv+E7TPUNueJgWPfPA2zrbun8S5KwLle3fY/ZZjAsc7nVmC1IfptBVxY133nBNIpI8fTFTAzcnk9vv5+ZS0Th/duuz756w4cT6EEeatQ9rnr+02DhYOePWreOtPZh3I8mlnz8iXgG7Xfm4Dzu6bzAsp+u8IweezMa7Wap87xZMTAkbI/fRk4C3hSQ9k7Cji0azr78Hjg/ilKcL58/XsFQwRxddtcXLv/nnK8PL+r3x9612FvOWvZP4fbJxjm+FiX+86usvNU4Dm1+xuU4/EywErAy0fa/j3r/CHKcXxZ4F8oFRlR+29LqQELyjHrXurxg8ePaQfWdbtN7b/mCOV3qMDxX4Ef9KR9D/hA7b4D2LSr3ybAXX2m38nXpDrWUgLPIbdN7f9B4PDefbyr/1F1/SbwW2DVrn47Ad+p3VfTE3xRLgrvruNeCcys6e+hBln197KU8/0+ffL4TeCDffqNFDi+suZh1WGGuZpyHp1HqeB6QVe/OXQFy5TALykXAs+lHDtWplxwfqaWkbOHmdeLgfnDldXJcvur2y7AgZl5S2bOo1zxvr3fwJl5YWael5kPZ+bVlIPoq0Yxv69m5p8y8z7KieaFI43Q5eeZeVaWW8DfpBzIDsnMhygnrlkRMa1r+JMy85LMvAf4T+BNURqTv41SSM/MzEcz82xKYdima9zjM/PSupwP9eRjM2DVOu8HM/MnlJ36LY3L8Sbg4Mycn5nXAgu1e8jMb2bmDTVvp1KuMl/Sb2KZeU5mXlyHv4hyAB9pm/xnZj6QmedSanI662ZnYL/MvKtu388yTHmgrIPbM/OvlEBouO35ELBRRKyemQsy87fDDHsL8PnMfKiugyuAbSNiHcp2en9m3pOZt1Bq3nbuGveGzPzvuu3uG3YtLGy4bd43T6OY/pvq+Ndm5nxKMDGcV1IChbuG6PcQ5YTzbMrJ9fLM7L011NFSXr+fmT/LzAcoJ/+XRsR6LQtVy80bgY/VbXIJ5RbfSN5POUltnplza9q7gP/IzOtqXj4O7FhvWZ0B/E29rQWlXJ6amQ82zOs+Ss1ca9vP5Sn70VqUW1b3NpS9E4C3dLV5eztwUu1+iBJkrV/Lz8+znjV6/BrYMCKeTNn+xwIzImJVyj59bmP+O0azf3aMdHx8FHhuRKycmTdm5qVdy7g+sG5m3p+Zo213d01mfjlLu80TKOtrHYDM/H5m/iWLcykn81d0jfsQ5Tz2UGaeSQkMnjXK+UPZT+7oSbuDsq8N1f8OYNUR2jlOqmNtZk7rt23qPv9OSo3ekDJzb8r6eAXljsgDddzVgE9SLpj6jXtIHffFlH2jsy5/BLwqynMVK/D4Re6T+kxqGuVid1HsBpyemXcPM8wulGB+fcq6Pqsrtvg/YJ+ImB4R/w94X01/Uj32/Q/l4vdpwKcp5/f3RcT7IuJnEfG1njjlLkrQ2ddkDBzXpVSTd1xT04YUEX8TEd+LiJsi4k5KQRmygWkfN3V130vZEVvd3NV9H3BrPch0ftMzvWu7uq+hnAzWphSGnSLi9s4f8HLKgWqocXutS6mafrRn+jMal2PdIfL2mIjYNSJ+35W35zLMOo6ITSPipxExLyLuoJx8h9smC2ow3T3/des4y/PE8jDcco1me76RcvK5JiLOjYiXDjPs9T0n1k4e1695vLFr/XyJUvvTMdy2G85I4/XLU6tht/sQtgHOHKpHDf6OoNyyuiUijomI1Yeb7wjl9bF81QPqfNqXbTql9nA0ywYlaDwyM7sf/lkf+N+ubXs55TbcOlnaA50KvK22QX0LjwdmLb4CrBMR2zUM+0xge+CArsB02LKXmedT9oHNI+LZdRpn1HE/Q6lZ/WFEXBkR+w4103qhM4cSJL6SEij+CngZixY4Lsrxtu/xsR433kw5xtwYEd+vywqlfVkAv4mISyPinxY1r5l5b+1cFSAito6I8yJifs3PNix8jLstF25TPtpzS8fdQO9+tDqPBym9/VcH7u5zEQBT41jb7fOUALw3eF5IZj5Sg8+ZlNphKBd5J9UgeLhxMzN/RzlnH1DT/kgJ6I6g1JauTbnV3u/BwAU8Hsw3i4gnUWpFh72wzcxfZuZ9mXlvZn6KUkvbuVA5mHIr//eUffPblED95jruYZn5gsx8M6Wy4GeU2G8vYAvKMa17/1+NJ16sLGQyBo43UA4UHU+raVCqX3t9kXK7bcMsjVw/QjlYLK576Lq6qFdk0xdzmt01Jk+jbNxbKSe4k+qVV+dvlXo11NHvQABl/azX8wDF0yi3wlrcOETeAIiI9Sm3x95DeSp7GuVWdmcdD5Wvr1NOUOtl5hqUW2LDbZM1I2KVnvnfQFk3nVqD7n6ty9XtCfnMzAsyc3vKifbblBrnfmb0XMV38ngt5Qp37a5tt3pmPme4eY+Ut8bx+uWpVd/t3kffwBEgM7+QmRsDG1Gaf3yw06tn0Jby+li+au3WWrQv2zzKLbnRLBuU9mQfjYg3dqVdS2nb1r1vrpSZnbyeQKkN2AK4NzN/3ZhHagB4AKX95UjHrMspt9F/EBGdmquWsncCpcbu7ZRajfvrvO/KzA9k5tMpDxX8W0Rs0Wfe51JuS78IuKD+3opy1+Fn/RZvhOUZjWGPj1nu+vwj5UL7j5TjFZl5U2a+IzPXpdRaHRVDvw5oVHmNiBUptTj/RbmAmEbZL8b8aWZK+9/n9+znz6/pnf4v6Or3gq5+Q5kKx9puWwCfqRVDnUD11xHx1j7DL8fjD9FsQalZ64y7HnBaRHy4YVwy8/TMfG5mPhnYn1Ljd0GfcS+iHPNG6/WUi+JzRjleUstbDSjfk5kz6v58G3Bhz4U59Q7FXpQmFM8FLspyJ+sCSpnq+FtKO+K+JmPg+A3KwXt6lEfTP0ZpIwAlgn5yfSKoYzVKG5e765XmvzA2/gSsFBHbRnm0/aOUdiGL420RsVG9yjiQciB/hLJ820V5D9SyEbFSrSKfOfzkHtOpWfhQRCwfEZtTnq47pXH804D9ImLNOs/3dvVbhVJI5wFExB6UQtdxMzAzHn/KD8o2mZ+Z99enuvrt5N0OiIgVIuIVwGuBb9Z1cxpwcESsVoPYf+Px8jAaC+WzzmuXiFij7jx3Um559fMUykFo+YjYibJznZnlduwPgc9GxOoRsUxEPCMiRtNc4mZKs4bR7o9D5mkU459Wx58ZEWuy8FXnQiJiA2DFzLy8T/+/qzXNnQco7ufx9Xkz0P1+xZbyuk1EvLxur09QGvw31dzWcvMt4OMR8aQor9HYrWHUSykNyI+MiNfVtKMp5W/9upzTI2L7rnn9ui7nZxldbWPHSZT2d7NHGjAzv0G5MP5RRDyjseydTDk5vY3Sho26HK+NiGfWgOQOSi1qv/J/LuUpy8tqsHsOpT3eVVmaEw2ld5svjr7Hx4hYJyK2r8HQA5QauEfrMu7UdQxdQDmODbWM82p6a35XoJwL5gEPR8TWlIuORdJZJkrgskxdvs7rVM6hbJv3RXntyntq+k/q/xMpQf+MiFiX8kDO8SPMcrIfa7v9DSUYfiGP3wrfjnIX4CkRsXNErFrX4VaUWv8f1+G2oJyrOuPeQLmAOLLuK++s57yo56l3d41LRGxcpzsdOAY4o9ZEDuVMeppj1WPbSpQ4a7m6XXvfc7obcOIwNcSdd6S+rK7HlSLig5Qa0F/W/jOivCYtImIzSjO4/YeY1OcobS3vpTyP8XdRLso3p7Tv7HgV5cnqviZj4HgQ5dbIRZSnnH5b0zrVx98Aroxyy2Jd4N8pgcldlCvNU8ciE7VqfG/K7aTrKSfDxX2H5EmUnfomysnifXVe11JuQ32EcjC6llJb07R96sF8O2BrypXjUcCuwxTyXgdQbktcRTkRPXYCzMzLKCfFX1MOCM+jFtjqJ5QT7k0R0XmVwN7AgRFxFyXwH+nq8ibKgf0G6isQuvL+Xsq6v5LSyPfrwHGNy9VtqHy+Hbg6ShOHd1Fqjvo5H9iQsn4PBnbMzNtqv10pJ5PL6nKczsLNDEbSeSn7bRExXDvL0eSpRedBiz9Q9rNvDTPstgwflK5ep7eAUpZuo9wOhdIubqO6z367sbx+nXLwm095AO5to1guKDXkq1LK1vHAV1tGysw/UE6mX64BweGU2vMf1vJ8HrBpz2gnUvaLkwEi4uiIOLpxfo9Q9pG1Goc/gXLR+ZOImMUIZa8eW35LCZp+3jWpDSntuO6m7NtHZeZP+8z2V5TG9Z3axcsoFwb9ahuhrLcdo7w/rvddcaMywvFxGUqAcwOlrLyKxysP/g44PyLupmzDfXKI93zWE+nBwC9rGd1shPzcRTl2n0ZZ52/l8SYAi+LtlNukX6TcfryPx2tNH6S8bmdXyu3Jf6I8aNhprvAlyqtiLqbcCfp+Tetn0h1ro7yX8BVPnAxkedbhps5fTb61NqFIyra+ri7Tf1Ha+55Rx72tZ9xHKLfqO20JX8/jbws5Gfjv+tdxOGWdX1Gn/45+C5ylzeYdEdF9bPgyZVu+hdJO+z662oxGxAxKTf6J9Og5hqxGKRsLKLHIbMpdkM6x/hmUffQeyh2GfTPzhz3TezUwLTP/t+b3N5Syci3wD9RXQdVAdxtGuHXeeUJMmhC1tunkzGytXR13EbE75QnRl090XjrGO09RXlh9RJaG/oOe1/GUp/o/OtKwk0FE7Ep5JcqkKR/dIuI4ygNaU2J9ajCmwrF2KouILYG9M3OHic7LooqI91KamH1ouOEm5Se2JE0651Ce5lOXKM1O9qbUmk46tVbyDZT2iZIGpNbyTenPMmbmf4881OS8VS1pMcTCn7Lq/hvydlCLzPx0ju5VQmMuIl7Rb9kmKD9bUW6d3ky5rTepRMQnKLcvP5OZV010fiQtGbxVLUmSpCbWOEqSJKmJgaMkSZKaLHUPx6y99to5a9asic6GJEnSiC688MJbM3NxP0AyZpa6wHHWrFnMmTNnorMhSZI0ooho+WTquPFWtSRJkpoYOEqSJKmJgaMkSZKaGDhKkiSpiYGjJEmSmhg4SpIkqYmBoyRJkpoYOEqSJKmJgaMkSZKaGDhKkiSpiYGjJEmSmix136oeD7P2/f4T0q4+ZNsJyIkkSdLYscZRkiRJTQwcJUmS1MTAUZIkSU0MHCVJktTEwFGSJElNDBwlSZLUxMBRkiRJTQwcJUmS1MTAUZIkSU0MHCVJktTEwFGSJElNDBwlSZLUxMBRkiRJTQwcJUmS1MTAUZIkSU0MHCVJktRk4IFjRCwbEb+LiO/V3xtExPkRMTciTo2IFWr6ivX33Np/Vtc09qvpV0TEVl3ps2va3IjYd9DLIkmStDQbjxrHfYDLu34fChyWmc8EFgB71vQ9gQU1/bA6HBGxEbAz8BxgNnBUDUaXBY4EtgY2At5Sh5UkSdIADDRwjIiZwLbAV+rvAF4NnF4HOQHYoXZvX39T+29Rh98eOCUzH8jMq4C5wEvq39zMvDIzHwROqcNKkiRpAAZd4/h54EPAo/X3k4HbM/Ph+vs6YEbtngFcC1D731GHfyy9Z5x+6ZIkSRqAgQWOEfFa4JbMvHBQ8xhFXvaKiDkRMWfevHkTnR1JkqQpaZA1ji8DXhcRV1NuI78aOByYFhHL1WFmAtfX7uuB9QBq/zWA27rTe8bpl/4EmXlMZm6SmZtMnz598ZdMkiRpKTSwwDEz98vMmZk5i/Jwy08ycxfgp8COdbDdgO/U7jPqb2r/n2Rm1vSd61PXGwAbAr8BLgA2rE9pr1DnccaglkeSJGlpt9zIg4y5DwOnRMRBwO+AY2v6scBJETEXmE8JBMnMSyPiNOAy4GHg3Zn5CEBEvAc4C1gWOC4zLx3XJZEkSVqKjEvgmJnnAOfU7ispT0T3DnM/sFOf8Q8GDh4i/UzgzDHMqiRJkvrwyzGSJElqYuAoSZKkJgaOkiRJamLgKEmSpCYGjpIkSWpi4ChJkqQmBo6SJElqYuAoSZKkJgaOkiRJamLgKEmSpCYGjpIkSWpi4ChJkqQmBo6SJElqYuAoSZKkJgaOkiRJamLgKEmSpCYGjpIkSWpi4ChJkqQmBo6SJElqYuAoSZKkJgaOkiRJamLgKEmSpCYGjpIkSWpi4ChJkqQmBo6SJElqYuAoSZKkJgaOkiRJamLgKEmSpCYGjpIkSWpi4ChJkqQmBo6SJElqYuAoSZKkJgaOkiRJamLgKEmSpCYGjpIkSWpi4ChJkqQmBo6SJElqYuAoSZKkJgaOkiRJamLgKEmSpCYGjpIkSWpi4ChJkqQmBo6SJElqYuAoSZKkJgaOkiRJamLgKEmSpCYGjpIkSWpi4ChJkqQmBo6SJElqYuAoSZKkJgaOkiRJajKwwDEiVoqI30TEHyLi0og4oKZvEBHnR8TciDg1Ilao6SvW33Nr/1ld09qvpl8REVt1pc+uaXMjYt9BLYskSZIGW+P4APDqzHwB8EJgdkRsBhwKHJaZzwQWAHvW4fcEFtT0w+pwRMRGwM7Ac4DZwFERsWxELAscCWwNbAS8pQ4rSZKkARhY4JjF3fXn8vUvgVcDp9f0E4Adavf29Te1/xYRETX9lMx8IDOvAuYCL6l/czPzysx8EDilDitJkqQBGGgbx1oz+HvgFuBs4C/A7Zn5cB3kOmBG7Z4BXAtQ+98BPLk7vWecfulD5WOviJgTEXPmzZs3FosmSZK01Blo4JiZj2TmC4GZlBrCZw9yfsPk45jM3CQzN5k+ffpEZEGSJGnKG5enqjPzduCnwEuBaRGxXO01E7i+dl8PrAdQ+68B3Nad3jNOv3RJkiQNwCCfqp4eEdNq98rAPwKXUwLIHetguwHfqd1n1N/U/j/JzKzpO9enrjcANgR+A1wAbFif0l6B8gDNGYNaHkmSpKXdciMPssieCpxQn35eBjgtM78XEZcBp0TEQcDvgGPr8McCJ0XEXGA+JRAkMy+NiNOAy4CHgXdn5iMAEfEe4CxgWeC4zLx0gMsjSZK0VBtY4JiZFwEvGiL9Skp7x970+4Gd+kzrYODgIdLPBM5c7MxKkiRpRH45RpIkSU0MHCVJktTEwFGSJElNDBwlSZLUxMBRkiRJTQwcJUmS1MTAUZIkSU0MHCVJktTEwFGSJElNDBwlSZLUxMBRkiRJTQwcJUmS1MTAUZIkSU0MHCVJktTEwFGSJElNDBwlSZLUxMBRkiRJTQwcJUmS1MTAUZIkSU0MHCVJktTEwFGSJElNDBwlSZLUZMTAMSJeFhGr1O63RcTnImL9wWdNkiRJk0lLjeMXgXsj4gXAB4C/ACcONFeSJEmadFoCx4czM4HtgSMy80hgtcFmS5IkSZPNcg3D3BUR+wFvA14ZEcsAyw82W5IkSZpsWmoc3ww8AOyZmTcBM4HPDDRXkiRJmnRaahx3Ar6amQsAMvOv2MZRkiRpqdNS47gOcEFEnBYRsyMiBp0pSZIkTT4jBo6Z+VFgQ+BYYHfgzxHxyYh4xoDzJkmSpEmk6QXg9anqm+rfw8CawOkR8ekB5k2SJEmTyIhtHCNiH2BX4FbgK8AHM/Oh+nT1n4EPDTaLkiRJmgxaHo5ZC3hDZl7TnZiZj0bEaweTLUmSJE02LW0c9wfWi4g9ACJiekRsUPtdPuD8SZIkaZJo+Vb1/sCHgf1q0vLAyYPMlCRJkiaflodjXg+8DrgHIDNvwE8OSpIkLXVaAscH61PVCRARqww2S5IkSZqMWgLH0yLiS8C0iHgH8CPK09WSJElairQ8Vf1Z4DXAncCzgI8BPxtkpiRJkjT5tASOx2bmPwFnA0TEqsCZwBaDzJgkSZIml5Zb1ddHxFEAEbEm8EN8qlqSJGmp0/Iex/8E7o6IoylB42cz86sDz5kkSZImlb63qiPiDV0/zwf+E/gNkBHxhsz81qAzJ0mSpMljuDaO2/X8/h3l5d/bUV7NY+AoSZK0FOkbOGbmHuOZEUmSJE1uLQ/HSJIkSQaOkiRJamPgKEmSpCYjvgA8IlYE3gjM6h4+Mw8cXLYkSZI02bR8OeY7wB3AhcADg82OJEmSJquWwHFmZs4e7YQjYj3gRGAdyut7jsnMwyNiLeBUSg3m1cCbMnNBRARwOLANcC+we2b+tk5rN+CjddIHZeYJNX1j4HhgZcpnEPfJzBxtXiVJkjSyljaOv4qI5y3CtB8GPpCZGwGbAe+OiI2AfYEfZ+aGwI/rb4CtgQ3r317AFwFqoLk/sCnwEmD/+ulD6jDv6Bpv1AGuJEmS2rQEji8HLoyIKyLiooi4OCIuGmmkzLyxU2OYmXcBlwMzgO2BE+pgJwA71O7tgROzOA+YFhFPBbYCzs7M+Zm5ADgbmF37rZ6Z59VaxhO7piVJkqQx1nKreuvFnUlEzAJeRPl04TqZeWPtdRPlVjaUoPLartGuq2nDpV83RLokSZIGYMTAMTOvWZwZRMSqwP8A78/MO0tTxsemnREx8DaJEbEX5fY3T3va0wY9O0mSpCXSQN/jGBHLU4LGr2Vm59vWN9fbzNT/t9T064H1ukafWdOGS585RPoTZOYxmblJZm4yffr0xVsoSZKkpdTAAsf6lPSxwOWZ+bmuXmcAu9Xu3Siv++mk7xrFZsAd9Zb2WcCWEbFmfShmS+Cs2ocyyKcAAA4cSURBVO/OiNiszmvXrmlJkiRpjLW0cVxULwPeDlwcEb+vaR8BDgFOi4g9gWuAN9V+Z1JexTOX8jqePQAyc35EfAK4oA53YGbOr9178/jreH5Q/yRJkjQAAwscM/MXQPTpvcUQwyfw7j7TOg44boj0OcBzFyObkiRJauS3qiVJktTEwFGSJElNDBwlSZLUxMBRkiRJTQwcJUmS1MTAUZIkSU0MHCVJktTEwFGSJElNDBwlSZLUxMBRkiRJTQwcJUmS1MTAUZIkSU0MHCVJktTEwFGSJElNDBwlSZLUxMBRkiRJTQwcJUmS1MTAUZIkSU0MHCVJktTEwFGSJElNDBwlSZLUxMBRkiRJTZab6AwsrWbt+/0npF19yLYTkBNJkqQ21jhKkiSpiYGjJEmSmhg4SpIkqYmBoyRJkpoYOEqSJKmJgaMkSZKaGDhKkiSpiYGjJEmSmhg4SpIkqYmBoyRJkpoYOEqSJKmJgaMkSZKaGDhKkiSpyXITnQENb9a+31/o99WHbDtBOZEkSUs7axwlSZLUxMBRkiRJTQwcJUmS1MQ2juOkt62iJEnSVGONoyRJkpoYOEqSJKmJgaMkSZKaGDhKkiSpiYGjJEmSmhg4SpIkqYmBoyRJkpoYOEqSJKmJgaMkSZKaGDhKkiSpycACx4g4LiJuiYhLutLWioizI+LP9f+aNT0i4gsRMTciLoqIF3eNs1sd/s8RsVtX+sYRcXEd5wsREYNaFkmSJA32W9XHA0cAJ3al7Qv8ODMPiYh96+8PA1sDG9a/TYEvAptGxFrA/sAmQAIXRsQZmbmgDvMO4HzgTGA28IMBLs+UMtS3sa8+ZNsJyIkkSVpSDKzGMTN/BszvSd4eOKF2nwDs0JV+YhbnAdMi4qnAVsDZmTm/BotnA7Nrv9Uz87zMTEpwugOSJEkamPFu47hOZt5Yu28C1qndM4Bru4a7rqYNl37dEOlDioi9ImJORMyZN2/e4i2BJEnSUmrCHo6pNYU5TvM6JjM3ycxNpk+fPh6zlCRJWuKMd+B4c73NTP1/S02/Hliva7iZNW249JlDpEuSJGlAxjtwPAPoPBm9G/CdrvRd69PVmwF31FvaZwFbRsSa9QnsLYGzar87I2Kz+jT1rl3TkiRJ0gAM7KnqiPgGsDmwdkRcR3k6+hDgtIjYE7gGeFMd/ExgG2AucC+wB0Bmzo+ITwAX1OEOzMzOAzd7U57cXpnyNLVPVEuSJA3QwALHzHxLn15bDDFsAu/uM53jgOOGSJ8DPHdx8ihJkqR2fjlGkiRJTQwcJUmS1MTAUZIkSU0G+clBjdJQnwkc7/n5WUJJktSPgaMWYjApSZL68Va1JEmSmhg4SpIkqYmBoyRJkprYxnGKsQ2iJEmaKNY4SpIkqYmBoyRJkpoYOEqSJKmJbRyXAOP94nBJkrR0ssZRkiRJTQwcJUmS1MTAUZIkSU1s47gUsS2kJElaHNY4SpIkqYmBoyRJkpoYOEqSJKmJbRw1aq3fy+4dzm9qS5I0tVnjKEmSpCbWOGpEg3wau7X2UpIkTTxrHCVJktTEwFGSJElNDBwlSZLUxDaOmvRsBylJ0uRg4Kgx4ecMJUla8hk4atIxCJUkaXIycNS4GcuA0JeQS5I0/gwctdSzDaUkSW0MHKUGBpeSJPk6HkmSJDUycJQkSVITA0dJkiQ1MXCUJElSEx+O0RLD9z9KkjRYBo7SGFrU90b61LYkaSowcNRSpbVWcqxqLw0IJUlLEgNHaRGN961xg1BJ0kTz4RhJkiQ1MXCUJElSE29VS0uYRX1AR5KkkRg4SuPMtpGSpKnKwFGawsYyCLWmUpI0Ets4SpIkqUlk5kTnYVxtsskmOWfOnIHOwy+YaEm1qLWQ3i6XpEUTERdm5iYTnY8Ob1VLWixjGRR6u1ySJjcDR0nNxvvLO63TNsCUpPFhG0dJkiQ1mfJtHCNiNnA4sCzwlcw8ZLjhbeMoaTSszZQ0kWzjOIYiYlngSOAfgeuACyLijMy8bGJzJmlJ0XohaIApaWkwpQNH4CXA3My8EiAiTgG2BwwcJY2rsbrTYAAqaTKb6oHjDODart/XAZtOUF4kabFNhqYuBq+S+pnqgWOTiNgL2Kv+vDsirhjwLNcGbh3wPPRErveJ4XqfGANb73HoIKa6xLC8T4yleb2vP9EZ6DbVA8frgfW6fs+saQvJzGOAY8YrUxExZzI1ZF1auN4nhut9YrjeJ4brfWK43iePqf46nguADSNig4hYAdgZOGOC8yRJkrREmtI1jpn5cES8BziL8jqe4zLz0gnOliRJ0hJpSgeOAJl5JnDmROejx7jdFtdCXO8Tw/U+MVzvE8P1PjFc75PElH8BuCRJksbHVG/jKEmSpHFi4DiGImJ2RFwREXMjYt+Jzs9UFxHrRcRPI+KyiLg0Ivap6WtFxNkR8ef6f82aHhHxhbr+L4qIF3dNa7c6/J8jYreJWqapJCKWjYjfRcT36u8NIuL8un5PrQ+kEREr1t9za/9ZXdPYr6ZfERFbTcySTB0RMS0iTo+IP0bE5RHxUsv74EXEv9ZjzCUR8Y2IWMnyPvYi4riIuCUiLulKG7PyHREbR8TFdZwvRESM7xIuJTLTvzH4ozyc8xfg6cAKwB+AjSY6X1P5D3gq8OLavRrwJ2Aj4NPAvjV9X+DQ2r0N8AMggM2A82v6WsCV9f+atXvNiV6+yf4H/BvwdeB79fdpwM61+2jgX2r33sDRtXtn4NTavVHdD1YENqj7x7ITvVyT+Q84Afjn2r0CMM3yPvB1PgO4Cli5/j4N2N3yPpB1/UrgxcAlXWljVr6B39Rho4679UQv85L4Z43j2Hns84eZ+SDQ+fyhFlFm3piZv63ddwGXUw7y21NOsNT/O9Tu7YETszgPmBYRTwW2As7OzPmZuQA4G5g9josy5UTETGBb4Cv1dwCvBk6vg/Su9872OB3Yog6/PXBKZj6QmVcBcyn7iYYQEWtQTqzHAmTmg5l5O5b38bAcsHJELAc8CbgRy/uYy8yfAfN7ksekfNd+q2fmeVmiyBO7pqUxZOA4dob6/OGMCcrLEqfeDnoRcD6wTmbeWHvdBKxTu/ttA7fN6H0e+BDwaP39ZOD2zHy4/u5eh4+t39r/jjq86310NgDmAV+tTQS+EhGrYHkfqMy8Hvgv4K+UgPEO4EIs7+NlrMr3jNrdm64xZuCoSS8iVgX+B3h/Zt7Z3a9eWfpqgDEUEa8FbsnMCyc6L0uZ5Si38b6YmS8C7qHcunuM5X3s1TZ121MC93WBVbCGdkJYvqcGA8ex0/T5Q41ORCxPCRq/lpnfqsk319sS1P+31PR+28BtMzovA14XEVdTmly8Gjiccquo8+7X7nX42Pqt/dcAbsP1PlrXAddl5vn19+mUQNLyPlivAa7KzHmZ+RDwLco+YHkfH2NVvq+v3b3pGmMGjmPHzx+Osdpu6Fjg8sz8XFevM4DOk3S7Ad/pSt+1Po23GXBHvQVyFrBlRKxZaxe2rGkaQmbul5kzM3MWpRz/JDN3AX4K7FgH613vne2xYx0+a/rO9SnUDYANKY3XNYTMvAm4NiKeVZO2AC7D8j5ofwU2i4gn1WNOZ71b3sfHmJTv2u/OiNisbsddu6alsTTRT+csSX+Up8D+RHma7j8mOj9T/Q94OeW2xUXA7+vfNpT2RD8G/gz8CFirDh/AkXX9Xwxs0jWtf6I0Vp8L7DHRyzZV/oDNefyp6qdTToRzgW8CK9b0lervubX/07vG/4+6Pa7AJxxb1vcLgTm1zH+b8tSo5X3w6/0A4I/AJcBJlCejLe9jv56/QWlH+hClhn3PsSzfwCZ1G/4FOIL6kRP/xvbPL8dIkiSpibeqJUmS1MTAUZIkSU0MHCVJktTEwFGSJElNDBwlSZLUxMBR0lIrIu4eof+0iNh7HPJxYES8ZoRhNo+Ivx90XiRpOAaOktTfNGDggWNmfiwzfzTCYJsDBo6SJpSBo6QpLyJmRcTlEfHliLg0In4YESsPMdwGEfHriLg4Ig7qSl81In4cEb+t/bavvQ4BnhERv4+IzwwzXO987o6Iw2pefhwR02v6CyPivIi4KCL+t375gog4PiJ2rN1XR8QBXfN4dkTMAt4F/GvNyysiYqeIuCQi/hARPxvL9SlJ/Rg4SlpSbAgcmZnPAW4H3jjEMIcDX8zM51G+YNFxP/D6zHwx8A/AZ+tny/YF/pKZL8zMDw4zXK9VgDk1L+cC+9f0E4EPZ+bzKV/D2H+IcQFurfP4IvDvmXk1cDRwWM3Lz4GPAVtl5guA1424diRpDBg4SlpSXJWZv6/dFwKzhhjmZZTPnkH5tFxHAJ+MiIsonz2bAawzxPitwz0KnFq7TwZeHhFrANMy89yafgLwyj7L8q0RlgPgl8DxEfEOYNk+w0jSmFpuojMgSWPkga7uR4An3KquhvrO6i7AdGDjzHwoIq6mfJN4UYdrmedwOsvyCH2O05n5rojYFNgWuDAiNs7M20Y5H0kaFWscJS1NfgnsXLt36UpfA7ilBoP/AKxf0+8CVmsYrtcywI61+63ALzLzDmBBRLyipr+dchu71UJ5iYhnZOb5mfkxYB6w3iimJUmLxBpHSUuTfYCvR8SHge90pX8N+G5EXAzMAf4IkJm3RcQvI+IS4AfAoUMNN4R7gJdExEeBW4A31/TdgKMj4knAlcAeo8j7d4HT6wM576U8KLMh5fb5j4E/jGJakrRIInO0d1AkScOJiLszc9WJzockjTVvVUuSJKmJNY6SJElqYo2jJEmSmhg4SpIkqYmBoyRJkpoYOEqSJKmJgaMkSZKaGDhKkiSpyf8Hc80y5QKg82EAAAAASUVORK5CYII=\n",
       "text/plain": [
-       "[('1286.TA.26-32-j19-1.12.H__8591182',\n",
-       "  array([   0, 1158,  306,  162,   94,   24,   28,   21,    3,    2,    0,\n",
-       "            1,    0,    0,    0,    0,    1,    0,    0,    0,    0,    0,\n",
-       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0])),\n",
-       " ('1286.TA.26-32-j19-1.12.H__8591184',\n",
-       "  array([  1, 762, 552, 292, 118,  48,  13,   8,   0,   1,   1,   0,   0,\n",
-       "           0,   0,   0,   0,   0,   0,   0,   0,   1,   0,   0,   0,   0,\n",
-       "           0,   0,   0,   0,   0,   0])),\n",
-       " ('1286.TA.26-32-j19-1.12.H__8591195',\n",
-       "  array([   0, 1083,  444,  143,   64,   35,   16,    9,    3,    1,    0,\n",
-       "            0,    0,    0,    0,    1,    0,    0,    0,    0,    1,    0,\n",
-       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0])),\n",
-       " ('1286.TA.26-32-j19-1.12.H__8591200',\n",
-       "  array([  2, 239, 227, 228, 212, 128,  74,  42,  29,  17,   3,   3,   2,\n",
-       "           0,   0,   0,   0,   0,   0,   0,   1,   0,   0,   0,   0,   0,\n",
-       "           0,   0,   0,   0,   0,   1])),\n",
-       " ('1286.TA.26-32-j19-1.12.H__8591209',\n",
-       "  array([   0, 1151,  308,  169,   94,   24,   29,   16,    4,    3,    1,\n",
-       "            0,    0,    0,    0,    0,    1,    0,    0,    0,    0,    0,\n",
-       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0]))]"
+       "<Figure size 720x360 with 1 Axes>"
       ]
      },
-     "execution_count": 1,
-     "metadata": {},
-     "output_type": "execute_result"
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
     }
    ],
    "source": [
-    "with gzip.open(\"../data/distributions.pickle\", \"rb\") as input_file:\n",
-    "    d = pickle.load(input_file)\n",
-    "\n",
-    "# Functon to take a slice from a dictionnary - head equivalent\n",
-    "def take(n, iterable):\n",
-    "    \"Return first n items of the iterable as a list\"\n",
-    "    return list(islice(iterable, n))\n",
-    "\n",
-    "# display a slice of it\n",
-    "take(5, d.items())"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Probability using cumulative distribution based on frequency of delays \n",
-    "\n",
-    "When we have __enough data__ and no ambiguity about `trip_id` and `stop_id` for a given distribution, then we can compute the probability $P(x \\leq X)$ for every x (delay in minute). \n",
-    "\n",
-    "Let's take a __threshold of 100__ sample points (=number of time we could measure a delay) as a minimum number of points to use this approach. \n",
-    "\n",
-    "_How many keys in our distionnary of distribution have at least this number of samples ?_"
+    "plot_data_points_hist(d_all)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
      "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAE/CAYAAADWuXIeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZxddX3/8deHhE32JVJJkKCiFqkLRMG68YMKAVRoBURR0KJoEcXWqqC2gkoFtS4IqCiUTVlEKynEIpXFlSW4sIpEFhPWQMImO35+f3y/l5zczJ25k8xkJjmv5+NxH3Pu92zfs7/PdicyE0mSJLXHSmNdAUmSJC1bBkBJkqSWMQBKkiS1jAFQkiSpZQyAkiRJLWMAlCRJapnlJgBGREbE88ZBPS6OiHeP0bhXj4j/iYj7I+J7fXS/XUTMXRZ1WxoRsU9E/His67GsRMSPImK/0RpGREyt28vEpRnHWImIhyLiOT3avTMift6j3bie7sHqPl5ExLPr/J8wSDd974sj4rCIOG3kajg8y8M8X5batq9dEhGxU0T8cKzrsTQi4gMRcdRQ3S11AKw7i87nLxHxSOP7Pj36WS6CyTi0B7ARsEFm7jmSAx7LHWVmficzd+yn2/G8Q+/3YJeZO2fmyUszrqUdRkRsPNxtMCJOiojPLuk4+5WZa2bmTaM9nqXRCJszu8pPi4jDxqhaSy0z/1Tn/1Mwtie8wzXWJwARsVdE/DIiHo6Iiwdo/9KIuLK2vzIiXtpoFxFxVETcWz9HRUSMdB3Hw742Ik7sPomo280dEfFARPyh1zoXEf9e+/27RtnnI2JO7ffWiPh4Vz9vjIhraib5ZURsMUQVjwCObPT/mYi4OiKe7N62I+LjXRnokZqDNuxR/1u6MtKPG+1WjYgvR8TtEbEgIo6LiJUb7b9Sy38VEVMa5W+LiKO7RvUtYJ+IeOZgE7rUAbDuLNbMzDWBPwFvbJR9Z2mHv6KqG/xw5/+mwB8y88nRqJP6tyQHmSVc5qNlF+B/x7oSK4BtIuJvx7oSGhfmA1+hER46ImIV4BzgNGA94GTgnFoOcACwO/AS4MXAG4H3LoM6L1MR8WrguQO0+hwwNTPXBt4EfDYitu7q97nAnsAdXf2eALyw9vu3lODzD7WfzYHvAO8D1gX+B5jRa/8dES8H1snMSxvFs4GPAud1d5+Z/9GVgY4CLs7MewaZDc2M1AzjhwDTgC2B5wNbAZ+s9XoFsDXwV8DPa7dExDrARzrdNer1KPAjYN9B6gGZOWIf4Bbg72rzqpSN4fb6+UotWwN4BPgL8FD9bAy8AvgVcB9lAR8DrNIYdgLP6zHei4HPAL8AHgR+DGxY220HzB2knocB36NsmA8CV9eZfyhwNzAH2LFrXJ8DLgceoGzU6zfabwv8sk7H74Dtuvo9otbzkYGmB/jr2t19wLXAm2r54cDjwBN1nu0/QL+rAycBC4DrKCvG3Eb7Q4A/1um8Dvj7xjgfBZ6qw76vlu8K/KZO5xzgsEGW/XbAXODjwD11Hu/TaL8OcAowD7iVssKuVNu9E/h517J+H3BjnQ/HAjFIPXep0/MgcBvwrz3q+M46748B7gd+D+zQVccTKOvfbcBngQld/X4ZuBf4bNewp3ctn9/1Wua17N391GmQ+d0cxgTgi3W+3wS8v87DiYP0/wPgHwYojzqNd9flfjVlh3RAnbbH6/T9z2Dra213EvAN4IK6bC4BNu1j2p7e1oENgBm1LpdTtvOf9+hvanO6gTdT1sMtKSe7nfX/XuAs6nZL2bF/oGtYV1G3jyHG9THgokb5afTYTlh8Pf8CZWe+Tq91D1iFEiz+ptHfM4GHgUnAhsC5df7PB35G3a66xn048LXavDLwZ+ALjf3Go8D6zXlIWW+fqu0eAo4ZbPvsMc2HAaf1uX98J2X9fRC4mbr/oGwzl1C2j3uAM3uM60+1bp3jyis785yyfSyow9250c+7gOvrOG8C3jvAPu3DlO3hDuBdfay/76aEgGbZjnW5Rld9p9fmXwIHNNrtD1y6vO5re9R7IuV48mIGP56/oM7rvbrK/7eO/xbq8XuAfidT9lkfrd8PAs5rtF+Jsh8ecB8L/Dvw7R7tem7btX3UdWi/QboZrO6zgD0b398GzKnNbwE+V5unAzNr8zHA23oMbx8a+6YBu+l34fW5gJ+eOODTwKWUndWkuoJ/prkCd/W7NWXnMJGyE7oe+FDXijpYAPwjJbitXr8fOci4mvU8rK7oO9Vxn0LZSXyCsqN8D3Bz17huoxxU1gC+T93B1ZXv3rqSrgS8vn6f1Oj3T8CL6rhW7qrXypSzjY9TdvzbUza0FzTqetog8/9IygFgfWAT4BoWDYB7UsL2SnWF+jPwrBxgx9CYd39Tu38xcBewe49xbwc8CXyJEvRfV4ffqfsplLC8Vl2+f6CG2O5x12V9LuWM7dmUHdn0Qep5B/Ca2rwesFWPOr6z1vGf67x+C+Wg0gkC/w18sy7XZ1ICx3u7+v1AXXarDzD8xZbPQMucxQNgzzoNsqybw3gfJThuUpf9RQwSAOt47gHWGqDdTsCVdd53DgSddeQkGsGXodfXk+r319Z14qvdy65H/ZoB8AxKWFuDss3d1msYLBpe3lXr1hnOwZT90ZRal28Cp9d2ewGXNYbzEsp2u8ogdeyMa61ap87+ZMgASNmevgWcDzyjj3XvOOCoxnAOZmEA/xwlZK9cP69hgDBWl83VtflvKfvLyxrtftc9D7vXs362z8G2CQbZP9bpfqCx7jwLeFFtPp2yP14JWA149VDLv2ueP0HZj08A/olyQSJq+10pV6SCss96mLr/YOE+7dN13u5S2683xPo7UAD8Z+BHXWXnAh+uzfcD2zTaTQMe7DH8Tr3G1b6WEiAHXDa1/UeAr3Zv4432x9X5m8CvgTUb7fYEzqnNt9AVoigndw/Vfm8CptTyg6hhqX6fQDneH9yjjt8DPtKj3VAB8LW1DmsO0s0tlOPoPMqFqpc02s2iEXopAS4pgX5Lyr5jdcqJ4xfqOnLBIOPaCpg/2Lo6mrej9gE+nZl3Z+Y8yhnoO3p1nJlXZualmflkZt5C2Rm+bhjj+6/M/ENmPkI5YLx0qB4afpaZ52e5tfo9yg7pyMx8gnIAmhoR6za6PzUzr8nMPwP/BuwV5aHpt1NWtpmZ+ZfMvICyUHdp9HtSZl5bp/OJrnpsC6xZx/14Zl5I2Tjf2ud07AUckZnzM3MOsMhzAZn5vcy8vdbtTMpZ3yt6DSwzL87Mq2v3V1F2xEMtk3/LzMcy8xLKlZXOvNkbODQzH6zL9z8ZZH2gzIP7MvNPlEAz2PJ8AtgiItbOzAWZ+etBur0b+EpmPlHnwQ3ArhGxEWU5fSgz/5yZd1OuhO3d6Pf2zPxaXXaPDDoXFjXYMu9Zp2EMf6/a/5zMnE8JBYN5LeWA/+AA7Z6gHDheSDlIXp+Z3bdcOvpZX8/LzJ9m5mOUg/grI2KTfiaqrjdvBv69LpNrKLfOhvIhysFmu8ycXcveB3wiM+fWuhwG7FFvBc0Anl9vF0FZL8/MzMf7GNcjlCtl/T4buTJlO1qfcivo4T7WvZOBtzaeCXsHcGptfoISljat68/Psu79u/wK2DwiNqAs/xOAyRGxJmWbvqTP+ncMZ/vsGGr/+Bdgy4hYPTPvyMxrG9O4KbBxZj6amcN9Lu3WzPxWlucaT6bMr40AMvO8zPxjFpdQDsqvafT7BOU49kRmzqQc4F8wzPFD2U7u7yq7n7KtDdT+fmDNIZ4DHFf72sxct9eyqdv8eylX2AaUmQdS5sdrKHcoHqv9rgX8B+XEp1e/R9Z+t6JsG515+X/A66K8d7AKC09Wn9FjUOtSTlqXxH7A2Zn50CDd7EMJ5ZtS5vX5jWzxv8DBETEpIv4K+GAtf0bd932fchL7bODzlOP7ByPigxHx04j4TldOeZASHnsazQC4MeXyc8ettWxAEfH8iDg3Iu6MiAcoC3zAByl7uLPR/DBlg+rXXY3mR4B76s6i852u4c1pNN9K2alvSFmoe0bEfZ0P8GrKDmegfrttTLnk+5eu4U/uczo2HqBuT4uIfSPit426bckg8zgitomIiyJiXkTcTzmIDrZMFtRQ3Bz/xrWflVl8fRhsuoazPN9MOYjcGhGXRMQrB+n2tq4DZKeOm9Y63tGYP9+kXI3pGGzZDWao/nrVqV+DLvcB7ALMHKhFDXHHUG4F3R0Rx0fE2oONd4j19el61R3jfPqftkmUq3nDmTYo4e/YzGy+5LIp8N+NZXs95fbWRlmelzkTeHt9RvOtLAxY/fg2sFFEvLGPbp8H7AYc3giYg657mXkZZRvYLiJeWIcxo/b7BcqVzh9HxE0RcchAI60nLLMoYe+1lMD3S+BVLFkAXJL9bc/9Y91vvIWyj7kjIs6r0wrl+asALo+IayPiH5e0rpn5cG1cEyAido6ISyNifq3PLiy6j7s3F33merjHlo6HgO7taG0Who3u9msDD/UI87B87GubvkIJ0t0heBGZ+VQNkVMoV2uhnKydWsPsYP1mZv6Gcsw+vJb9nhLMjqFcvdyQcgu71wtwC1gYyvsWEc+gXKUc9AQ1M3+RmY9k5sOZ+TnKVdPOCccRlFvkv6Vsmz+kBO67ar9fzsyXZOZbKCf9P6VkuAOAHSj7tOb2vxaLn3QsYjQD4O2UDb7j2bUMymXNbl+n3MbaPMvDnB+nbPRL68800n49Q5q0lMNsXsF4NmUh3UM5UJ1az4Q6nzXq2UlHrw0ayvzZpOtFgWdTbjH1444B6gZARGxKue10EOUt4nUpt4g783igen2XcqDZJDPXodxqGmyZrBcRa3SN/3bKvOmcxTfb9TtdTYvVMzOvyMzdKAfMH1KuAPcyueusulPHOZQzzg0by27tzHzRYOMeqm599terTv3qudx76BkAATLz6MzcGtiC8ljFRzqtujrtZ319ul71atP69D9t8yi3uoYzbVCet/pkRLy5UTaH8uxXc9tcLTM7dT2Zcna+A/BwZv6qzzpSg9zhlOcTh9pnXU+5Pf2jiOhcSepn3TuZcgXtHZSrDI/WcT+YmR/OzOdQHp7/l4jYoce4L6Hc7n0ZcEX9vhPlLsBPe03eENMzHIPuH7PchXk95YT595T9FZl5Z2a+JzM3plxFOi4G/hmaYdU1IlalXFX5IuVEYF3KdjHib99Sno99cdd2/uJa3mn/kka7lzTaDWR52Nc27QB8oV7g6QTOX0XE23p0P5GFL4vsQLnS1el3E+CsiPhYH/2SmWdn5paZuQHwKcoVuCt69HsVZZ83XH9PObm9eJj9JXV9q8HwoMycXLfne4Eru06wqXcMDqA8mrAlcFWWO0tXUNapjr+mPGfb02gGwNMpO+FJUV6J/nfKPXQoiXaD+gZLx1qUZ0Aeqmd+/8TI+AOwWkTsGuWV6k9SnptYGm+PiC1q6v80ZYf8FGX63hjld4QmRMRq9dLzlMEH97TOmf5HI2LliNiO8jbYGX32fxZwaESsV8f5gUa7NSgr2zyAiHgXZeXpuAuYEgvfSoOyTOZn5qP1LaReG2vT4RGxSkS8BngD8L06b84CjoiItWoY/RcWrg/DsUg967j2iYh16kbwAOVWUi/PpOxMVo6IPSkbycwstzl/DPxnRKwdEStFxHMjYjiPIdxFeVxguNvVgHUaRv9n1f6nRMR6LHoWuIiI2AxYNTOv79H+5fXKb+dFgUdZOD/vApq/z9fP+rpLRLy6Lq/PUB5s7+tKal1vfgAcFhHPiPLzDfv10eu1lAelj42IN9Wyb1DWv03rdE6KiN0a4/pVnc7/ZHhX/zpOpTyfNn2oDjPzdMoJ7v9FxHP7XPdOoxxk3k55xos6HW+IiOfVYHE/5apmr/X/EspbgdfV0Hox5Xm1m7M8pjOQ7mW+NHruHyNio4jYrYaaxyhXxP5Sp3HPxj50AWU/NtA0zqvl/dZ3FcqxYB7wZETsTDl5WCKdaaIEkJXq9HV+xuNiyrL5YJSf+zioll9Y/55CCe+TI2JjyosnJw0xyvG+r216PiXUvpSFt5jfSLkq/8yI2Dsi1qzzcCfKVfif1O52oByrOv3eTjkROLZuK++tx7yox6n3N/olIrauw50EHA/MqFcGBzKTrsec6r5tNUpemliXa/fvZO4HnDLIFdvOb2y+qs7H1SLiI5Qrkr+o7SdH+XmuiIhtKY+XfWqAQX2J8iziw5T3FV4e5eR6O8rzjx2vo7wJ3NNoBsDPUm45XEV5K+fXtaxzWfZ04KYotwI2Bv6VEjAepJz5nTkSlaiXnA+k3Ka5jXJQW9rfIDyVsnHeSdnpf7COaw7l9s7HKTuVOZSrJ33N57pTfiOwM+VM7jhg30FW1m6HUy7330w5oDx9IMvM6ygHt19RNuy/oa541YWUA+edEdF5hf1A4NMR8SAlwA91tncnZQd9O/XV+0bdP0CZ9zdRHmb9LnBin9PVNFA93wHcEuXRgfdRruT0chmwOWX+HgHskZn31nb7Ug4K19XpOJtFb98PpfPj3PdGxGDPIQ6nTv3ovFDwO8p29oNBut2VwcPl2nV4Cyjr0r2U24xQnhvbom6zP+xzff0uZSc2n/Ki19uHMV1QrlivSVm3TgL+q5+eMvN3lIPit+qB/auUq9k/ruvzpcA2Xb2dQtkuTgOIiG9ExDf6HN9TlG1k/T67P5ly8nhhRExliHWv7lt+TQk/P2sManPKc04PUbbt4zLzoh6j/SXlIfLO1b7rKAG/19U/KPNtjyi/P9b9W2PDMsT+cSVKULmdsq68joUXAV4OXBYRD1GW4cE5wO9E1gPiEcAv6jq67RD1eZCy7z6LMs/fxsJb60viHZTbj1+n3NZ7hIVXMR+n/MzLvpTbfv9IeaGu8xjANyk/UXI15c7MebWsl3G3r43yu3avWXwwkOVdgDs7n1p8T300ISnLem6dpi9SnoedUfu9t6vfpyi3wDvP2v09C3/d4jTga/XT8VXKPL+hDv89vSY4yzON90dEc9/wLcqyfCvlOeZHaDxTGRGTKVfWT6FL1z5kLcq6sYCSRaZT7kp09vXPpWyjf6Zc8T8kM3/cNbztgXUz879rfS+nrCtzgP9H/QmiGlh3YYhb0p03oaSlUq/+nJaZ/V7tXOYi4p2UNxpfPdZ16VjWdYryw8XHZHmgfbTHdRLlLfRPDtXteBAR+1J+imPcrB9NEXEi5UWk5WJ+anQsD/va5VlE7AgcmJm7j3VdllREfIDy6NZHB+tuXP7LJEmj5mLK22dqiPI4x4GUq5jjTr1K+A+U5/ckjZJ61W25/nd5mfm1obtajv4XsNQ2sei/GGp+BrzN0o/M/HwO7ydsRlxEvKbXtI1RfXai3JK8i3K7bFyJiM9Qbgt+ITNvHuv6SFoxeAtYkiSpZbwCKEmS1DIGQEmSpJZZbl8C2XDDDXPq1KljXQ1JkqQhXXnllfdk5tL+I4oRs9wGwKlTpzJr1qyxroYkSdKQIqKff2W5zHgLWJIkqWUMgJIkSS1jAJQkSWoZA6AkSVLLGAAlSZJaxgAoSZLUMgZASZKkljEASpIktYwBUJIkqWUMgJIkSS1jAJQkSWqZ5fZ/AS8LUw85b7GyW47cdQxqIkmSNHK8AihJktQyBkBJkqSWMQBKkiS1jAFQkiSpZQyAkiRJLWMAlCRJahkDoCRJUssYACVJklrGAChJktQyBkBJkqSWMQBKkiS1jAFQkiSpZQyAkiRJLWMAlCRJahkDoCRJUssYACVJklqm7wAYERMi4jcRcW79vllEXBYRsyPizIhYpZavWr/Pru2nNoZxaC2/ISJ2apRPr2WzI+KQkZs8SZIkdRvOFcCDgesb348CvpyZzwMWAPvX8v2BBbX8y7U7ImILYG/gRcB04LgaKicAxwI7A1sAb63dSpIkaRT0FQAjYgqwK/Dt+j2A7YGzaycnA7vX5t3qd2r7HWr3uwFnZOZjmXkzMBt4Rf3MzsybMvNx4IzarSRJkkZBv1cAvwJ8FPhL/b4BcF9mPlm/zwUm1+bJwByA2v7+2v3T5V399CqXJEnSKBgyAEbEG4C7M/PKZVCfoepyQETMiohZ8+bNG+vqSJIkLZf6uQL4KuBNEXEL5fbs9sBXgXUjYmLtZgpwW22+DdgEoLZfB7i3Wd7VT6/yxWTm8Zk5LTOnTZo0qY+qS5IkqduQATAzD83MKZk5lfISx4WZuQ9wEbBH7Ww/4JzaPKN+p7a/MDOzlu9d3xLeDNgcuBy4Ati8vlW8Sh3HjBGZOkmSJC1m4tCd9PQx4IyI+CzwG+CEWn4CcGpEzAbmUwIdmXltRJwFXAc8Cbw/M58CiIiDgPOBCcCJmXntUtRLkiRJgxhWAMzMi4GLa/NNlDd4u7t5FNizR/9HAEcMUD4TmDmcukiSJGnJ+J9AJEmSWsYAKEmS1DIGQEmSpJYxAEqSJLWMAVCSJKllDICSJEktYwCUJElqGQOgJElSyxgAJUmSWsYAKEmS1DIGQEmSpJYxAEqSJLWMAVCSJKllDICSJEktYwCUJElqGQOgJElSyxgAJUmSWsYAKEmS1DIGQEmSpJYxAEqSJLWMAVCSJKllDICSJEktYwCUJElqGQOgJElSyxgAJUmSWsYAKEmS1DIGQEmSpJYxAEqSJLWMAVCSJKllDICSJEktYwCUJElqGQOgJElSyxgAJUmSWsYAKEmS1DIGQEmSpJYxAEqSJLWMAVCSJKllDICSJEktYwCUJElqGQOgJElSyxgAJUmSWsYAKEmS1DIGQEmSpJYxAEqSJLWMAVCSJKllDICSJEktYwCUJElqGQOgJElSyxgAJUmSWsYAKEmS1DIGQEmSpJYZMgBGxGoRcXlE/C4iro2Iw2v5ZhFxWUTMjogzI2KVWr5q/T67tp/aGNahtfyGiNipUT69ls2OiENGfjIlSZLU0c8VwMeA7TPzJcBLgekRsS1wFPDlzHwesADYv3a/P7Cgln+5dkdEbAHsDbwImA4cFxETImICcCywM7AF8NbarSRJkkbBkAEwi4fq15XrJ4HtgbNr+cnA7rV5t/qd2n6HiIhafkZmPpaZNwOzgVfUz+zMvCkzHwfOqN1KkiRpFPT1DGC9Uvdb4G7gAuCPwH2Z+WTtZC4wuTZPBuYA1Pb3Axs0y7v66VU+UD0OiIhZETFr3rx5/VRdkiRJXfoKgJn5VGa+FJhCuWL3wlGtVe96HJ+Z0zJz2qRJk8aiCpIkScu9Yb0FnJn3ARcBrwTWjYiJtdUU4LbafBuwCUBtvw5wb7O8q59e5ZIkSRoF/bwFPCki1q3NqwOvB66nBME9amf7AefU5hn1O7X9hZmZtXzv+pbwZsDmwOXAFcDm9a3iVSgviswYiYmTJEnS4iYO3QnPAk6ub+uuBJyVmedGxHXAGRHxWeA3wAm1+xOAUyNiNjCfEujIzGsj4izgOuBJ4P2Z+RRARBwEnA9MAE7MzGtHbAolSZK0iCEDYGZeBbxsgPKbKM8Ddpc/CuzZY1hHAEcMUD4TmNlHfSVJkrSU/E8gkiRJLWMAlCRJahkDoCRJUssYACVJklrGAChJktQyBkBJkqSWMQBKkiS1jAFQkiSpZQyAkiRJLWMAlCRJahkDoCRJUssYACVJklrGAChJktQyBkBJkqSWMQBKkiS1jAFQkiSpZQyAkiRJLWMAlCRJahkDoCRJUssYACVJklrGAChJktQyBkBJkqSWMQBKkiS1jAFQkiSpZQyAkiRJLWMAlCRJahkDoCRJUssYACVJklrGAChJktQyBkBJkqSWMQBKkiS1jAFQkiSpZQyAkiRJLWMAlCRJahkDoCRJUssYACVJklrGAChJktQyBkBJkqSWMQBKkiS1jAFQkiSpZQyAkiRJLWMAlCRJahkDoCRJUssYACVJklrGAChJktQyBkBJkqSWMQBKkiS1jAFQkiSpZQyAkiRJLWMAlCRJapkhA2BEbBIRF0XEdRFxbUQcXMvXj4gLIuLG+ne9Wh4RcXREzI6IqyJiq8aw9qvd3xgR+zXKt46Iq2s/R0dEjMbESpIkqb8rgE8CH87MLYBtgfdHxBbAIcBPMnNz4Cf1O8DOwOb1cwDwdSiBEfgUsA3wCuBTndBYu3lPo7/pSz9pkiRJGsiQATAz78jMX9fmB4HrgcnAbsDJtbOTgd1r827AKVlcCqwbEc8CdgIuyMz5mbkAuACYXtutnZmXZmYCpzSGJUmSpBE2rGcAI2Iq8DLgMmCjzLyjtroT2Kg2TwbmNHqbW8sGK587QLkkSZJGQd8BMCLWBL4PfCgzH2i2q1fucoTrNlAdDoiIWRExa968eaM9OkmSpBVSXwEwIlamhL/vZOYPavFd9fYt9e/dtfw2YJNG71Nq2WDlUwYoX0xmHp+Z0zJz2qRJk/qpuiRJkrr08xZwACcA12fmlxqtZgCdN3n3A85plO9b3wbeFri/3io+H9gxItarL3/sCJxf2z0QEdvWce3bGJYkSZJG2MQ+unkV8A7g6oj4bS37OHAkcFZE7A/cCuxV280EdgFmAw8D7wLIzPkR8RngitrdpzNzfm0+EDgJWB34Uf1IkiRpFAwZADPz50Cv3+XbYYDuE3h/j2GdCJw4QPksYMuh6iJJkqSl538CkSRJahkDoCRJUssYACVJklrGAChJktQyBkBJkqSWMQBKkiS1jAFQkiSpZQyAkiRJLWMAlCRJahkDoCRJUssYACVJklrGAChJktQyBkBJkqSWMQBKkiS1jAFQkiSpZQyAkiRJLWMAlCRJahkDoCRJUssYACVJklrGAChJktQyBkBJkqSWMQBKkiS1zMSxrsDybuoh5y1WdsuRu45BTSRJkvrjFUBJkqSWMQBKkiS1jAFQkiSpZQyAkiRJLWMAlCRJahkDoCRJUssYACVJklrGAChJktQyBkBJkqSWMQBKkiS1jAFQkiSpZQyAkiRJLWMAlCRJapmJY12Btph6yHmLfL/lyF3HqCaSJKntvAIoSZLUMgZASZKkljEASpIktYzPAA5T97N8kiRJyxuvAEqSJLWMAVCSJKllDICSJEktYwCUJElqGQOgJElSyxgAJUmSWsYAKEmS1DIGQEmSpJYxAEqSJLWMAVCSJKllhgyAEXFiRNwdEWUUnZgAAAmISURBVNc0ytaPiAsi4sb6d71aHhFxdETMjoirImKrRj/71e5vjIj9GuVbR8TVtZ+jIyJGeiIlSZK0UD//C/gk4BjglEbZIcBPMvPIiDikfv8YsDOwef1sA3wd2CYi1gc+BUwDErgyImZk5oLazXuAy4CZwHTgR0s/aSuGgf738C1H7joGNZEkSSuKIa8AZuZPgfldxbsBJ9fmk4HdG+WnZHEpsG5EPAvYCbggM+fX0HcBML22WzszL83MpITM3ZEkSdKoWdJnADfKzDtq853ARrV5MjCn0d3cWjZY+dwBygcUEQdExKyImDVv3rwlrLokSVK7LfVLIPXKXY5AXfoZ1/GZOS0zp02aNGlZjFKSJGmFs6QB8K56+5b69+5afhuwSaO7KbVssPIpA5RLkiRplCxpAJwBdN7k3Q84p1G+b30beFvg/nqr+Hxgx4hYr74xvCNwfm33QERsW9/+3bcxLEmSJI2CId8CjojTge2ADSNiLuVt3iOBsyJif+BWYK/a+UxgF2A28DDwLoDMnB8RnwGuqN19OjM7L5YcSHnTeHXK27++ASxJkjSKhgyAmfnWHq12GKDbBN7fYzgnAicOUD4L2HKoekiSJGlk+J9AJEmSWsYAKEmS1DIGQEmSpJbp51/BaZgG+vdty3p8/rs4SZLUiwFwBWUolCRJvXgLWJIkqWUMgJIkSS1jAJQkSWoZnwEcIz6jJ0mSxopXACVJklrGAChJktQyBkBJkqSW8RnAcWRZ/4C0JElqJ68ASpIktYwBUJIkqWUMgJIkSS3jM4DLIZ8VlCRJS8MrgJIkSS1jAJQkSWoZA6AkSVLL+Axgi/X7/4i7u/N/FkuStHzzCqAkSVLLeAWwRUbz7eF+ryZKkqSx5xVASZKkljEASpIktYwBUJIkqWV8BlDLjM8JSpI0PhgAtQj/zZwkSSs+A6BGjWFSkqTxyQCoYRvJYOePUUuStOwZALXC8BlDSZL6YwBUqxgSJUnyZ2AkSZJaxwAoSZLUMgZASZKkljEASpIktYwvgWjc8fcDJUkaXQZAaQBL+ruDvmUsSVoeGAC1XOr3KuFIXU002EmSViQGQLXesr7lbJiUJI01XwKRJElqGQOgJElSy3gLWBqnlvRFFEmShmIAlJaQzw5KkpZXBkBpHBjJMOmVQ0nSUHwGUJIkqWUiM8e6Dktk2rRpOWvWrFEdh/+RQiuqJb0q6G1oSVoyEXFlZk4b63p0eAtYEjCy4c7b0JI0vhkApRZa1v9Jpd9hGxQladnwGUBJkqSWGTfPAEbEdOCrwATg25l55GDd+wygpOHw6qKkseQzgAOIiAnAscDrgbnAFRExIzOvG9uaSVpR9HtCZ1CU1AbjIgACrwBmZ+ZNABFxBrAbYACUtEyN1JV/g6Sk8Wy8BMDJwJzG97nANmNUF0laauPhERJDqKRexksA7EtEHAAcUL8+FBE3jPIoNwTuGeVxaHHO97HhfB8bozbf46jRGOoKw/V9bLR5vm861hVoGi8B8DZgk8b3KbVsEZl5PHD8sqpURMwaTw9stoXzfWw438eG831sON/HhvN9/BgvPwNzBbB5RGwWEasAewMzxrhOkiRJK6RxcQUwM5+MiIOA8yk/A3NiZl47xtWSJElaIY2LAAiQmTOBmWNdjy7L7HazFuF8HxvO97HhfB8bzvex4XwfJ8bND0FLkiRp2RgvzwBKkiRpGTEADiAipkfEDRExOyIOGev6LO8iYpOIuCgirouIayPi4Fq+fkRcEBE31r/r1fKIiKPr/L8qIrZqDGu/2v2NEbHfWE3T8iQiJkTEbyLi3Pp9s4i4rM7fM+uLV0TEqvX77Np+amMYh9byGyJip7GZkuVHRKwbEWdHxO8j4vqIeKXr++iLiH+u+5hrIuL0iFjN9X3kRcSJEXF3RFzTKBux9Tsito6Iq2s/R0dELNspbInM9NP4UF5C+SPwHGAV4HfAFmNdr+X5AzwL2Ko2rwX8AdgC+DxwSC0/BDiqNu8C/AgIYFvgslq+PnBT/btebV5vrKdvvH+AfwG+C5xbv58F7F2bvwH8U20+EPhGbd4bOLM2b1G3g1WBzer2MWGsp2s8f4CTgXfX5lWAdV3fR32eTwZuBlav388C3un6Pirz+rXAVsA1jbIRW7+By2u3UfvdeayneUX8eAVwcU//W7rMfBzo/Fs6LaHMvCMzf12bHwSup+ysd6McKKl/d6/NuwGnZHEpsG5EPAvYCbggM+dn5gLgAmD6MpyU5U5ETAF2Bb5dvwewPXB27aR7vneWx9nADrX73YAzMvOxzLwZmE3ZTjSAiFiHcoA8ASAzH8/M+3B9XxYmAqtHxETgGcAduL6PuMz8KTC/q3hE1u/abu3MvDRLGjylMSyNIAPg4gb6t3STx6guK5x6m+VlwGXARpl5R211J7BRbe61DFw2w/cV4KPAX+r3DYD7MvPJ+r05D5+ev7X9/bV75/vwbAbMA/6r3nr/dkSsgev7qMrM24AvAn+iBL/7gStxfV9WRmr9nlybu8s1wgyAWmYiYk3g+8CHMvOBZrt6pucr6SMoIt4A3J2ZV451XVpmIuX22Ncz82XAnym3xJ7m+j7y6jNnu1EC+MbAGnjFdEy4fi8fDICL6+vf0ml4ImJlSvj7Tmb+oBbfVS/3U//eXct7LQOXzfC8CnhTRNxCeZRhe+CrlFswnd8Abc7Dp+dvbb8OcC/O9+GaC8zNzMvq97MpgdD1fXT9HXBzZs7LzCeAH1C2Adf3ZWOk1u/banN3uUaYAXBx/lu6EVafqzkBuD4zv9RoNQPovPm1H3BOo3zf+vbYtsD99dbC+cCOEbFePdvfsZZpAJl5aGZOycyplPX4wszcB7gI2KN21j3fO8tjj9p91vK961uTmwGbUx7S1gAy805gTkS8oBbtAFyH6/to+xOwbUQ8o+5zOvPd9X3ZGJH1u7Z7ICK2rctx38awNJLG+i2U8fihvLX0B8rbX58Y6/os7x/g1ZTbAVcBv62fXSjP2/wEuBH4P2D92n0Ax9b5fzUwrTGsf6Q8lD0beNdYT9vy8gG2Y+FbwM+hHNBmA98DVq3lq9Xvs2v75zT6/0RdHjfgG3n9zO+XArPqOv9DyluOru+jP98PB34PXAOcSnmT1/V95Ofz6ZTnLJ+gXPHefyTXb2BaXYZ/BI6h/tMKPyP78T+BSJIktYy3gCVJklrGAChJktQyBkBJkqSWMQBKkiS1jAFQkiSpZQyAkiRJLWMAlCRJahkDoCRJUsv8fynmF0ZdtOUIAAAAAElFTkSuQmCC\n",
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoQAAAFNCAYAAACZuH6uAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3debglVXn3/e+PySiDgHR4GG00ZMCoiDxI4kReDWMQR9SoIDFBE0k0MQPGRFBjgjFqYlQMPhJAnDDRiIJRNIojyBBkVGmhEbCZkUFQQe/3j7UObA5n2Ifuc0531/dzXec6tVdNq1atWnXvWlW1U1VIkiRpuNZZ7AxIkiRpcRkQSpIkDZwBoSRJ0sAZEEqSJA2cAaEkSdLAGRBKkiQN3FoZECapJL+0GuTjS0l+f5HW/eAkn0pyS5KPjTH9HkmuWoi8rYwkL0ryucXOx0JJ8pkkB8/XMpIs7cfLeiuzjsWS5PYkj5hm3EuTfHWacav1ds+U99VFku17+a87wzRjt8VJjkxy4qrL4dysCWW+kIbW1k4nyYeTPHOx87EyknwzyaNmm25BA8LeeEz8/TzJnSOfXzTNPGtEoLIaei6wJfCwqnreqlzwYjacVfXBqtpznGlX5wZ+3JNfVe1TVcevzLpWdhlJtp7rMZjkuCR/90DXOa6q2qiqLpvv9ayMkeDz1EnpJyY5cpGytdKq6vu9/H8Gi/sFeK4W+wtBkgOTfD3JHUm+NMX4nZOc08efk2TnkXFJ8pYkN/a/tyTJqs7jYrW1SbZKcnKSH/R9tHTS+IsmxRJ3J/nUyPj9k1zYx309yU4j4x6U5B192TcneU+S9WfIy2OAxwKfHCdvI/NtnuT62cokySOSfDrJbUluSPKPI/l8f5Ir+rjzkuwzMt92Sc5IclOSt01a5meS7DppVf8EvHGmvMACB4S98dioqjYCvg/sP5L2wYXMy5qkNwBz3VcPB75bVXfPR540vgdy0nmA+3y+7Av892JnYi3whCS/udiZ0GrhJuCfgaMmj0iyAS0AORHYDDge+GRPBzgUeCYtUHkMsD/w8gXI80L5Oa29ec5UI6vqUSNxxMbAlcDHAJLsCHwQeAWwKfAp4OSRNvhwYFfg14FfBnYB/maGvLwc+GDd+wseM+ZtxFuAS2aaoO/P04D/Af4PsC1tnwOs17frqcBDex5PGglAX0urFzsAz5wIAJM8H7i8qs6etLqTgd9K8n9mzHVVLcofsBx4eh9+EO3g+EH/++eetiFwJ20n3N7/tgZ2A74B/BBYAbwL2GBk2QX80jTr/RLwJuBrwG3A54At+rg9gKtmyOeRtIp3Yp/3Alqlei1wXd+Be05a1z8A3wRupR3km4+M3x34et+ObwF7TJr3zT2fd061PcCv9el+CFwEPKOnvwH4KXBXL7OXTTHvg4HjgJuBi4G/GN122oHzvb6dFwPPGlnnj4Gf9WX/sKfvB/xv384rgSNn2Pd7AFcBfw3c0Mv4RSPjHwqcAFwPXEE7GNbp414KfHXSvn4FcGkvh3cDmSGf+/btuQ24GvjzafL40l727wJuAb4NPG1SHt9Pq39XA38HrDtp3ncANwJ/N2nZe0/aP9+abp/3tN8fJ08zlPfoMtalfVu8AbgMeGUvw/VmmP/jwLOnSE/fxuv6fr+A1tAe2rftp337PjVTfe3jjgPeS2sgbwNOBx4+xrbdc6wDD6M1fLfSjrk3jdaVSfMtHd1uWgO/vOd/He6t/zcCJ9GPW+AU4I8nLet8+vExy7r+CvjiSPqJTHOccP96/lbgq73eTVn3gA1ogcajR+b7ReAOYAmwBfDpXv43AV+hH1eT1v0G4F/78PrAj4C3jrQbPwY2Hy1DWr39WR93O/CumY7Pabb5SODEMdvHl9Lq723A5fT2g3bMnE47Pm4APjrNur7f8zZxXvmNiTKnHR839+XuMzLPIbST/G193S+fok17De14WAEcMkb9/X3gS5PS9uz7NZPyu3cf/jpw6Mi4lwFnrKlt7Qxls15f59IZpnlqX/6G/fNhwCkj49ehtaVP65/PBp43Mv53gStnWP5lwJPmkjfgN2nxySFM0/706Q4FvjKH8jgfeE4f/gzwK334I8CBwCa0c/Cm08x/GnDwjOuYyw5alX/cN9B6I3AGrfFa0iv8m0Yr9KR5H09rLNajNUqXAK+eVHFnCgi/RwvkHtw/HzXDukbzeWSv+Hv1dZ9AazReR2s4/4AWnY+u62raSWZD4D/pDR6wDe1ks2+vtL/dPy8Zmff7wKP6utaflK/1gWW0A30D4P+jHRi/MpLXE2co/6NoJ4TNge2AC7lvQPg8WvC9DvB82klhqz7upZMrei+7R/fpHwNcCzxzmnXvAdwNvJ0W+D+1L38i7yfQgueN+/79Lj2onbzuvq8/Tfs2uD2tYdt7hnyuAJ7chzcDdpkmjy/tefzTXtbPp51kJgKDTwD/1vfrL9ICkJdPmveP+7578BTLv9/+mWqfc/+AcNo8zbCvR5fxCloguV3f919khoCwr+cGYOMpxu0FnNPLfuLEMFFHjmMkEGb2+npc//yUXif+ZfK+myZ/owHhR2jB24a0Y+7q6ZbBfYOZQ3reJpbzKlp7tG3Py78BH+7jDgTOHFnOY2nH7QYz5HFiXRv3PE20J7MGhLTj6X3AZ4GHjFH33gO8ZWQ5r+LegPwfaEH3+v3vyUwRnPV9c0Ef/k1ae3nmyLhvTS7DyfVsnONzpmOCGdrHvt23jtSdrYBH9eEP09rjdYBfYIqT+VR5Hynzu2jt+LrAH9IuUKSP3w94JK2uP5UWaO8yqU17Yy/bffv4zWapv1MFhH8KfGZS2qeB1/ThW4AnjIzbFbhtmuVP5Gu1amtpAeWU+2ZkmnECwmOB40Y+HwacOvJ5Xdo5+1X989nAgSPjX9TX8dAplr1hH7dk3Lz19Z1Li1HuVyZT5P0DtODuBtrx8+hppt2yb8ev9s9v7du6KS1AfxStzTx4hvW9E3j7TGW+unRJvQh4Y1VdV1XX076hvmS6iavqnKo6o6rurqrltMbxqXNY379X1Xer6k7aCWTn2WYY8ZWq+my1rtiP0Rqoo6rqLtoJaWmSTUem/0BVXVhVPwL+Fjgw7SbsF9Mq7qlV9fOqOo1WWfcdmfe4qrqob+ddk/KxO7BRX/dPq+p/aAfrC8fcjgOBN1fVTVV1Ja2y3KOqPlZVP+h5+yit0u023cKq6ktVdUGf/nxawzzbPvnbqvpJVZ1Ou/IyUTYvAF5bVbf1/fs2ZqgPtDL4YVV9nxbgzLQ/7wJ2SrJJVd1cVefOMO11wD9X1V29DL4D7JdkS9p+enVV/aiqrqNdKXvByLw/qKp/7fvuzhlL4b5m2ufT5mkOyz+wz39lVd1ECxJm8hRaAHDbFOPuop1IfpV20rykqlZMs5xx6uspVfXlqvoJ7aT+G0m2G2ejer15DvD6vk8upHWpzObVtKvje1TVsp72CuB1VXVVz8uRwHN7t9PJwC/3rilo9fKjVfXTMdZ1J+1K2rj3Vq5PO442p91ec8cYde944IUj95S9hHbSgba/tqJdeb2rqr5S/UwxyTeAHZM8jLb/3w9sk2Qj2jF9+pj5nzCX43PCbO3jz4FfT/LgqlpRVReNbOPDga2r6sdVNdf72q6oqvdVuy/yeFp5bQlQVadU1feqOZ3Wu/TkkXnvop3H7qqqU2lXy35ljuuHdpzcMintFtqxNtX4W4CNZrmPcLVqa6tq0wewb+4jyUNo98ofN5L8eeCpac8ebMC9X0Af0sf/N/CqJEt69+mf9PSHcH8T5/Gp2r7p/Anty9M5Y0y7La3830m7+HIK9701AIB+j+MHgeOr6ts9+R9ode902pfADWgXYj6V5ENJvpzksEnru21km6a0ugSEW9MuV0+4oqdNKckv9xsxr0lyK/D3tO6QcV0zMnwH7QAb17Ujw3cCN/TGY+Izk5Z35cjwFbRGfgtao/W8JD+c+AOeRGuAppp3sq1pl7p/Pmn524y5HVtPkbd7JDmo38g6kbdfZ4YyTvKEJF/sN9LeQjupzrRPbu5B8uj6t+7zrM/968NM2zWX/fkc2knliiSnJ/mNGaa9etIJcyKPD+95XDFSPv9Gu1ozYaZ9N5PZ5psuT+Oacb9PYV/g1KlG9KDuXbSuo+uSHJNkk5nWO0t9vSdfVXU7rVtz3G1bwr333YwufzZ/Aby7qkYfmnk48ImRfXsJrTtsy6r6MfBR4MX9Hs8Xcm/ANY7/B2yZZP8xpv0l4ADgDSMB54x1r6rOpB0DeyT51b6Mk/u8b6VdCf1cksuSHD7VSvsXmLNpwd9TaCedrwNP5IEFhA+kvZ22feztxvNpbcyKJKf0bQX4S9oVvG+mPXzwew80r1V1Rx/cCCDJPiM38v+QdmyMtnE31n3v2Z7ruWXC7bTuv1GbcG9gMnn8JsDt0wT3sGa0tQ/Es2ltxD31sQdMB9PapRW0bbyY1m0O7QvZ/wLn0er0f9EC19Hz+oQf9v8bTzHufpJsTQsIXzdm/u+kXUH8TD++/4l228uvjSxzHVr78lPaFUEA+oWc51fVY2lXBv+V1iN1OK237+nAK5Lcs6y+HT9kBqtLQPgDWgMwYfueBu2y7GRH07q9dqyqTWjfAlbFU1Y/YuSbQv8GtWQllzl6hWN7WuW7gXbi+kD/pjTxt2FVjd5kPN0BDq18tpv04MH2tC6pcayYIm8AJHk4rZvqMNpTypvSKtlEGU+Vrw/RTjzbVdVDaV1TM+2TzZJsOGn9P6CVzcS3/NFx427XqPvls6rOqqoDaCfQ/6JdIZ7ONpO+dU/k8UrgJ7R7Tyf23SZVNfpY/0z7bqbxs803XZ7GNe1+n8a0ASFAVb2zqh4P7ES7DeMvJkZNmnSc+npPvvrVqM0Zf9uup3WNzWXboN2v9TdJRm8Sv5J279josfkLVTWR1+NpvRpPA+6oqm+MmUd6w/8G2v2Ns7VZl9C6sz+TZOJK0zh173jaFbaXAP/Rg1j6VaDXVNUjgGcAf5bkadOs+3Ra9/DjgLP6571ovQRfnm7zZtmeuZixfazWS/PbtC/Q36a1V1TVNVX1B1W1Ne2BgPdk6tfezCmvSR5Eu+Xnn2hfDDalHRer/Ole2v21j5l0nD+mp0+Mf+zIuMeOjJvKmtDWPhAHAydMDoSr6j+q6ter6mHAEbSu8LP6uDur6rCq2qYfBzcC50z6ojqxnB9x7+1l49iNVh8vTnINLVDbrV+4murVTOczQz3s+//9tCvUz6mpe4yg3Yt4Ru8VeTRwdm9nLuifJ/wa7V7caa0uAeGHaY3ykiRbAK/n3qdtrgUeluShI9NvTLuH5Pb+zfAPV1E+vgv8QpL9+mXav6Hdd7EyXpxkp355+420BvpntO3bP8leSdZN8gv9Mve2Yy534krAXyZZP8ketKfNPjLm/CcBr02yWV/nH4+Mm7h34nqAJIfQrhBOuBbYdtKl7Y2Bm6rqx0l2o92sO5s3JNkgyZOB3wE+1svmJODNSTbuwemfcW99mIv75LOv60VJHtoPrltpXU/T+UXgT3r5Po92QJ1arVv0c8DbkmySZJ0kj0wyl9sWrqXdXjDXY3DKPM1h/pP6/Nsm2Yz2jXJKSXYAHlRVUz4tl+T/9ivDEw8e/Jh7y/NaYPT9gOPU132TPKnvrzfRGrmxrrT2evNx4MgkD0l71cTBY8x6Ee0hn3cneUZPey+t/j28b+eSJAeMrOsbfTvfxtyuDk74AO3+tr1nm7CqPkz7wvv5JI8cs+6dCDyLFhSeMJGY5HeS/FI/0dxCu+o5Xf0/HTgIuLifXL5Eu9/t8mq39Uxl8j5fGdO2j0m2THJAD3J+Qrti9vO+jc8baUNvprVjU23j9T193PxuQDsXXA/cnfYKkLFeyTKViW2iXdVep2/fxOtPvkTbN3+S9vqRiStD/9P/n0AL5rfpV6Vew327Taeyure199HLZuLc+6D+eXT8tsBvMcVtIUke38t3CXAMcPJEV+tEmaXZnXYb1xEzZOVUJt36NEPePkMLPnfuf6+nXY3ceaQXcdSJwO5Jnt4DxlfTgvSJ9vZoWvu+f01z21GSX6Q9GHhkT7qc9jTxRrR7Sy8byfPjaQ+WTGt1CQj/jtZFcT4tqj23p01cAv4wcFla18HWwJ/TAo7baN8MP7oqMlFVtwB/ROvWuZp2klvZdyB+gHawXkM7CfxJX9eVtO6gv6Y1MlfSrq6MtU96I70/sA+tEr0HOGjkHoPZvIHWPXA57QRzz4mtqi6mney+QTvQH017unXC/9BOpNckuaGn/RHwxiS30Q6E2b4NXkNrsH9Af03ASN7/mFb2l9FurP8Q7QbcuZoqny8BlqfdavAK2pWe6ZwJ7Egr3zcDz62qG/u4g2gniYv7dvwH9+3un83Ey8JvTDLTfYxzydM4Jh5Q+BbtOPv4DNPux8zB5iZ9eTfT6tKNtG5JaN9sd+rH7H+NWV8/RGucb6I1Xi+ew3ZBu6K9Ea1uHQf8+zgzVdW3aCfJ9/UT/b/QrnZ/rtfnM4AnTJrtBNpxcSJAkvcmee+Y6/sZ7RjZfMzpj6d9mfyftNdOzFj3ettyLi0Y+srIonak3WN1O+3Yfk9VfXGa1X6d9tDdxNXAi2kB/3RXB6GV23PT3u/2zhmmm9Us7eM6tMDlB7S68lTuvSjwf4Ezk9xO24evqineU9m7g98MfK3X0d1nyc9ttLb7JFqZ/y73dsU/EC+hdRkeTbsX7E7uvcr5U9prZQ6idfH9Hu0BvYnbBv6N9jqVC2g9N6f0tOmsdm1t2jsCn3z/xdzjTlo9hXYFeHJA9BLgG1X1vSnm/RdauX2Htt1/MDLukbS6/SNaMHl4Vc308u1jgBf1L1Ez5q3aPZrXTPzRvnTd1YdHX+S+fZ/+O7Q27r09nwfQ3rzw0x6cv5wWWF6T6d/X/E+0+1Yn8vMPtCv7V9IeJpt4/cz+tIeXZuxxmXh6Slow/erQiVU17tXQBZfkpbQnJp+02HmZsNB5SnuR8ruq3SA/3+s6jvaU+0zvBFttJDmI9uqP1aZ+jEpyLO3BpjWiPDU/1oS2dnWX5EPASVX1X4udlwcqyZm0p8cvnGm61fJnmyStFr5Ee5JQI9Ju//gj2lXO1U6/ivhs2v1/klZCVY1z+9Nqraom93BMaXXpMpa0EnLfn3Ia/ZupW2ZGVfWP0927slCSPHm6bVuk/OxF68K8lta9tlpJ8iZaN+Jbq+ryxc6PpDWHXcaSJEkD5xVCSZKkgTMglCRJGri18qGSLbbYopYuXbrY2ZAkSZrVOeecc0NVrewPYayUtTIgXLp0KWefffbsE0qSJC2yJOP81Oa8sstYkiRp4AwIJUmSBs6AUJIkaeAMCCVJkgbOgFCSJGngDAglSZIGzoBQkiRp4AwIJUmSBs6AUJIkaeAMCCVJkgbOgFCSJGng1srfMpZW1tLDT1mU9S4/ar9FWa8kadi8QihJkjRwBoSSJEkDZ0AoSZI0cAaEkiRJA2dAKEmSNHAGhJIkSQNnQChJkjRwBoSSJEkDZ0AoSZI0cAaEkiRJA2dAKEmSNHAGhJIkSQNnQChJkjRwBoSSJEkDZ0AoSZI0cAaEkiRJA2dAKEmSNHDzFhAm2S7JF5NcnOSiJK/q6UcmuTrJef1v35F5XptkWZLvJNlrJH3vnrYsyeHzlWdJkqQhWm8el3038JqqOjfJxsA5SU7r495RVf80OnGSnYAXAI8CtgY+n+SX++h3A78NXAWcleTkqrp4HvMuSZI0GPMWEFbVCmBFH74tySXANjPMcgDwkar6CXB5kmXAbn3csqq6DCDJR/q0BoSSJEmrwILcQ5hkKfA44MyedFiS85Mcm2SznrYNcOXIbFf1tOnSJ6/j0CRnJzn7+uuvX8VbIEmStPaa94AwyUbAfwKvrqpbgaOBRwI7064gvm1VrKeqjqmqXatq1yVLlqyKRUqSJA3CfN5DSJL1acHgB6vq4wBVde3I+PcBn+4frwa2G5l9257GDOmSJElaSfP5lHGA9wOXVNXbR9K3GpnsWcCFffhk4AVJHpRkB2BH4JvAWcCOSXZIsgHtwZOT5yvfkiRJQzOfVwifCLwEuCDJeT3tr4EXJtkZKGA58HKAqrooyUm0h0XuBl5ZVT8DSHIY8FlgXeDYqrpoHvMtSZI0KPP5lPFXgUwx6tQZ5nkz8OYp0k+daT5JkiQ9cP5SiSRJ0sAZEEqSJA2cAaEkSdLAGRBKkiQNnAGhJEnSwBkQSpIkDZwBoSRJ0sAZEEqSJA2cAaEkSdLAGRBKkiQNnAGhJEnSwBkQSpIkDZwBoSRJ0sAZEEqSJA2cAaEkSdLAGRBKkiQNnAGhJEnSwBkQSpIkDZwBoSRJ0sAZEEqSJA2cAaEkSdLAGRBKkiQNnAGhJEnSwBkQSpIkDZwBoSRJ0sAZEEqSJA2cAaEkSdLAGRBKkiQNnAGhJEnSwBkQSpIkDZwBoSRJ0sAZEEqSJA2cAaEkSdLAGRBKkiQNnAGhJEnSwBkQSpIkDZwBoSRJ0sAZEEqSJA3cvAWESbZL8sUkFye5KMmrevrmSU5Lcmn/v1lPT5J3JlmW5Pwku4ws6+A+/aVJDp6vPEuSJA3RfF4hvBt4TVXtBOwOvDLJTsDhwBeqakfgC/0zwD7Ajv3vUOBoaAEkcATwBGA34IiJIFKSJEkrb94CwqpaUVXn9uHbgEuAbYADgOP7ZMcDz+zDBwAnVHMGsGmSrYC9gNOq6qaquhk4Ddh7vvItSZI0NAtyD2GSpcDjgDOBLatqRR91DbBlH94GuHJktqt62nTpkiRJWgXWm+8VJNkI+E/g1VV1a5J7xlVVJalVtJ5DaV3NbL/99qtikZLWUksPP2XB17n8qP0WfJ2SNK55vUKYZH1aMPjBqvp4T762dwXT/1/X068GthuZfdueNl36fVTVMVW1a1XtumTJklW7IZIkSWux+XzKOMD7gUuq6u0jo04GJp4UPhj45Ej6Qf1p492BW3rX8meBPZNs1h8m2bOnSZIkaRWYzy7jJwIvAS5Icl5P+2vgKOCkJC8DrgAO7ONOBfYFlgF3AIcAVNVNSd4EnNWne2NV3TSP+ZYkSRqUeQsIq+qrQKYZ/bQppi/gldMs61jg2FWXO0mSJE3wl0okSZIGzoBQkiRp4AwIJUmSBs6AUJIkaeAMCCVJkgbOgFCSJGngDAglSZIGzoBQkiRp4AwIJUmSBs6AUJIkaeAMCCVJkgbOgFCSJGngDAglSZIGzoBQkiRp4NZb7AysyZYefsqirHf5UfstynolSdLaySuEkiRJA2dAKEmSNHAGhJIkSQNnQChJkjRwBoSSJEkDZ0AoSZI0cAaEkiRJA2dAKEmSNHAGhJIkSQNnQChJkjRwBoSSJEkDZ0AoSZI0cAaEkiRJA2dAKEmSNHAGhJIkSQNnQChJkjRwBoSSJEkDZ0AoSZI0cAaEkiRJAzdrQJjkiUk27MMvTvL2JA+f/6xJkiRpIYxzhfBo4I4kjwVeA3wPOGFecyVJkqQFM05AeHdVFXAA8K6qejew8fxmS5IkSQtlvTGmuS3Ja4EXA09Jsg6w/vxmS5IkSQtlnCuEzwd+Arysqq4BtgXeOq+5kiRJ0oIZJyB8HvDvVfUVgKr6flXNeg9hkmOTXJfkwpG0I5NcneS8/rfvyLjXJlmW5DtJ9hpJ37unLUty+Nw2T5IkSbMZJyDcEjgryUk9OMuYyz4O2HuK9HdU1c7971SAJDsBLwAe1ed5T5J1k6wLvBvYB9gJeGGfVpIkSavIrAFhVf0NsCPwfuClwKVJ/j7JI2eZ78vATWPm4wDgI1X1k6q6HFgG7Nb/llXVZVX1U+AjfVpJkiStIuM8VEJVVZJrgGuAu4HNgP9IclpV/eUc13lYkoOAs4HXVNXNwDbAGSPTXNXTAK6clP6EqRaa5FDgUIDtt99+jlmSVg9LDz9lUda7/Kj9FmW9kqTVwzgvpn5VknOAfwS+Bjy6qv4QeDzwnDmu72jgkcDOwArgbXOcf1pVdUxV7VpVuy5ZsmRVLVaSJGmtN84Vws2BZ1fVFaOJVfXzJL8zl5VV1bUTw0neB3y6f7wa2G5k0m17GjOkS5IkaRUY5x7CI4DtkhwCkGRJkh36uEvmsrIkW418fBYw8QTyycALkjyoL3tH4JvAWcCOSXZIsgHtwZOT57JOSZIkzWzWK4RJjgB2BX4F+HfaS6lPBJ44y3wfBvYAtkhyFXAEsEeSnYEClgMvB6iqi5KcBFxMu0fxlVX1s76cw4DPAusCx1bVRXPeSkmSJE1rnC7jZwGPA84FqKofJJn1p+uq6oVTJL9/hunfDLx5ivRTgVPHyKckSZIegHHeQ/jT/lvGBZBkw/nNkiRJkhbSOAHhSUn+Ddg0yR8Anwf+3/xmS5IkSQtlnC7jtwFPB26l3Uf4euDL85kpSZIkLZxxAsL3V9XvAacBJNmIdk/f0+YzY5IkSVoY43QZX53kPQBJNgM+R3vKWJIkSWuBcd5D+LfA7UneSwsG31ZV/z7vOZMkSdKCmLbLOMmzRz6eCfwt7WXRleTZVfXx+c6cJEmS5t9M9xDuP+nz/9JeSr0/7RU0BoQDsvTwUxZlvcuP2m9R1itJ0pBMGxBW1SELmRFJkiQtjnEeKpEkSdJazIBQkiRp4AwIJUmSBm7WF1MneRDwHGDp6PRV9cb5y5YkSZIWyji/VPJJ4BbgHOAn85sdSZIkLbRxAsJtq2rvec+JJEmSFsU49xB+Pcmj5z0nkiRJWhTjXCF8EvDSJJfTuowDVFU9Zl5zJkmSpAUxTkC4z7znQpIkSYtm1oCwqq5YiIxIkiRpcfgeQkmSpIEzIJQkSRo4A0JJkqSBMyCUJEkaOANCSZKkgTMglCRJGjgDQkmSpIEzIJQkSRo4A0JJkqSBMyCUJEkaOANCSZKkgTMglCRJGjgDQkmSpIEzIJQkSRo4A0JJkqSBMyCUJEkaOANCSZKkgTMglCRJGjgDQkmSpIGbt4AwybFJrkty4Uja5klOS3Jp/79ZT0+SdyZZluT8JLuMzHNwn/7SJAfPV34lSZKGaj6vEB4H7D0p7XDgC1W1I/CF/hlgH2DH/ncocDS0ABI4AngCsBtwxEQQKUmSpFVj3gLCqsAkl04AAA1hSURBVPoycNOk5AOA4/vw8cAzR9JPqOYMYNMkWwF7AadV1U1VdTNwGvcPMiVJkrQSFvoewi2rakUfvgbYsg9vA1w5Mt1VPW26dEmSJK0ii/ZQSVUVUKtqeUkOTXJ2krOvv/76VbVYSZKktd5CB4TX9q5g+v/revrVwHYj023b06ZLv5+qOqaqdq2qXZcsWbLKMy5JkrS2WuiA8GRg4knhg4FPjqQf1J823h24pXctfxbYM8lm/WGSPXuaJEmSVpH15mvBST4M7AFskeQq2tPCRwEnJXkZcAVwYJ/8VGBfYBlwB3AIQFXdlORNwFl9ujdW1eQHVSRJkrQS5i0grKoXTjPqaVNMW8Arp1nOscCxqzBrkiRJGuEvlUiSJA2cAaEkSdLAGRBKkiQNnAGhJEnSwBkQSpIkDZwBoSRJ0sAZEEqSJA2cAaEkSdLAGRBKkiQNnAGhJEnSwBkQSpIkDZwBoSRJ0sAZEEqSJA2cAaEkSdLAGRBKkiQNnAGhJEnSwBkQSpIkDZwBoSRJ0sAZEEqSJA2cAaEkSdLAGRBKkiQNnAGhJEnSwBkQSpIkDZwBoSRJ0sCtt9gZkLT4lh5+yqKsd/lR+y3KeiVJ9+UVQkmSpIEzIJQkSRo4A0JJkqSBMyCUJEkaOANCSZKkgTMglCRJGjgDQkmSpIEzIJQkSRo4A0JJkqSBMyCUJEkaOANCSZKkgTMglCRJGjgDQkmSpIEzIJQkSRq4RQkIkyxPckGS85Kc3dM2T3Jakkv7/816epK8M8myJOcn2WUx8ixJkrS2WswrhL9VVTtX1a798+HAF6pqR+AL/TPAPsCO/e9Q4OgFz6kkSdJabHXqMj4AOL4PHw88cyT9hGrOADZNstViZFCSJGlttFgBYQGfS3JOkkN72pZVtaIPXwNs2Ye3Aa4cmfeqnnYfSQ5NcnaSs6+//vr5yrckSdJaZ71FWu+TqurqJL8InJbk26Mjq6qS1FwWWFXHAMcA7LrrrnOaV5IkacgW5QphVV3d/18HfALYDbh2oiu4/7+uT341sN3I7Nv2NEmSJK0CCx4QJtkwycYTw8CewIXAycDBfbKDgU/24ZOBg/rTxrsDt4x0LUuSJGklLUaX8ZbAJ5JMrP9DVfXfSc4CTkryMuAK4MA+/anAvsAy4A7gkIXPsiRpLpYefsqirHf5UfstynqlNd2CB4RVdRnw2CnSbwSeNkV6Aa9cgKxJkiQN0ur02hlJkiQtAgNCSZKkgTMglCRJGjgDQkmSpIEzIJQkSRo4A0JJkqSBW6yfrpOkRXtXnSTpvrxCKEmSNHAGhJIkSQNnQChJkjRwBoSSJEkDZ0AoSZI0cAaEkiRJA2dAKEmSNHC+h1CS1mK+61HSOLxCKEmSNHAGhJIkSQNnl7FWa3Z3SZI0/7xCKEmSNHAGhJIkSQNnQChJkjRwBoSSJEkDZ0AoSZI0cAaEkiRJA2dAKEmSNHAGhJIkSQPni6klSVpJi/US/eVH7bco69Xax4BQkiSt9gy655cBoSRpreHPXUoPjAHhGsgGT5IkrUo+VCJJkjRwBoSSJEkDZ5exJC0Ab/XQfFiMejWUhyyGxiuEkiRJA2dAKEmSNHAGhJIkSQNnQChJkjRwBoSSJEkDt8YEhEn2TvKdJMuSHL7Y+ZEkSVpbrBEBYZJ1gXcD+wA7AS9MstPi5kqSJGntsKa8h3A3YFlVXQaQ5CPAAcDFi5orSZIGxndqrp3WiCuEwDbAlSOfr+ppkiRJWklryhXCWSU5FDi0f7w9yXcWYLVbADcswHrWdJbT+Cyr8VhO47OsxmdZjWdQ5ZS3rNTs45bVw1dqLavAmhIQXg1sN/J52552j6o6BjhmITOV5Oyq2nUh17kmspzGZ1mNx3Ian2U1PstqPJbT+NakslpTuozPAnZMskOSDYAXACcvcp4kSZLWCmvEFcKqujvJYcBngXWBY6vqokXOliRJ0lphjQgIAarqVODUxc7HJAvaRb0Gs5zGZ1mNx3Ian2U1PstqPJbT+NaYskpVLXYeJEmStIjWlHsIJUmSNE8MCB8Af0bvXkm2S/LFJBcnuSjJq3r6kUmuTnJe/9t3ZJ7X9rL7TpK9Fi/3Cy/J8iQX9DI5u6dtnuS0JJf2/5v19CR5Zy+r85Pssri5XzhJfmWk7pyX5NYkr7ZeQZJjk1yX5MKRtDnXoSQH9+kvTXLwYmzLfJumrN6a5Nu9PD6RZNOevjTJnSN1670j8zy+H7fLenlmMbZnPk1TVnM+3tb28+M05fTRkTJanuS8nr5m1amq8m8Of7SHWr4HPALYAPgWsNNi52sRy2MrYJc+vDHwXdrPCx4J/PkU0+/Uy+xBwA69LNdd7O1YwPJaDmwxKe0fgcP78OHAW/rwvsBngAC7A2cudv4XqczWBa6hvadr8PUKeAqwC3DhA61DwObAZf3/Zn14s8XetgUqqz2B9frwW0bKaunodJOW881efunluc9ib9sCldWcjrchnB+nKqdJ498GvH5NrFNeIZy7e35Gr6p+Ckz8jN4gVdWKqjq3D98GXMLMvyJzAPCRqvpJVV0OLKOV6ZAdABzfh48HnjmSfkI1ZwCbJtlqMTK4yJ4GfK+qrphhmsHUq6r6MnDTpOS51qG9gNOq6qaquhk4Ddh7/nO/sKYqq6r6XFXd3T+eQXuv7bR6eW1SVWdUO5OfwL3lu9aYpl5NZ7rjba0/P85UTv0q34HAh2daxupapwwI586f0ZtGkqXA44Aze9JhvVvm2IkuLCy/Aj6X5Jy0X9cB2LKqVvTha4At+/DQy2rCC7hvA2u9ur+51qGhl9eE36NdnZmwQ5L/TXJ6kif3tG1o5TNhaGU1l+Nt6PXqycC1VXXpSNoaU6cMCLVKJNkI+E/g1VV1K3A08EhgZ2AF7TK64ElVtQuwD/DKJE8ZHdm/Lfrof5f2IvpnAB/rSdarWViHxpPkdcDdwAd70gpg+6p6HPBnwIeSbLJY+VtNeLzNzQu575fXNapOGRDO3aw/ozc0SdanBYMfrKqPA1TVtVX1s6r6OfA+7u2+G3T5VdXV/f91wCdo5XLtRFdw/39dn3zQZdXtA5xbVdeC9WoGc61Dgy6vJC8Ffgd4UQ+g6d2fN/bhc2j3wv0yrVxGu5UHU1YP4HgbbL1Ksh7wbOCjE2lrWp0yIJw7f0ZvRL9n4v3AJVX19pH00XvdngVMPJF1MvCCJA9KsgOwI+3m2rVekg2TbDwxTLu5/UJamUw85Xkw8Mk+fDJwUH9SdHfglpFuwaG4zzdu69W05lqHPgvsmWSz3g24Z09b6yXZG/hL4BlVdcdI+pIk6/bhR9Dq0GW9vG5Nsntv7w7i3vJdqz2A423I58enA9+uqnu6gte4OrXYT7WsiX+0J/e+S4v2X7fY+VnksngSrXvqfOC8/rcv8AHggp5+MrDVyDyv62X3HVaDJ6sWsKweQXvq7lvARRN1B3gY8AXgUuDzwOY9PcC7e1ldAOy62NuwwOW1IXAj8NCRtMHXK1qAvAK4i3bv0cseSB2i3T+3rP8dstjbtYBltYx2n9tEe/XePu1z+nF5HnAusP/IcnalBUPfA95F/1GHtelvmrKa8/G2tp8fpyqnnn4c8IpJ065RdcpfKpEkSRo4u4wlSZIGzoBQkiRp4AwIJUmSBs6AUJIkaeAMCCVJkgbOgFDSWi/J7bOM3zTJHy1APt6Y5OmzTLNHkt+c77xI0igDQkmCTYF5Dwir6vVV9flZJtsDMCCUtKAMCCWtMZIsTXJJkvcluSjJ55I8eIrpdkjyjSQXJPm7kfSNknwhybl93AF91FHAI5Ocl+StM0w3eT23J3lHz8sXkizp6TsnOSPJ+Uk+0X8NhCTHJXluH16e5A0j6/jVJEuBVwB/2vPy5CTPS3Jhkm8l+fKqLE9JmmBAKGlNsyPw7qp6FPBD2q8BTPYvwNFV9WjarwpM+DHwrKraBfgt4G39p6MOB75XVTtX1V/MMN1kGwJn97ycDhzR008A/qqqHkP7pYcjppgX4Ia+jqOBP6+q5cB7gXf0vHwFeD2wV1U9FnjGrKUjSQ+AAaGkNc3lVXVeHz4HWDrFNE/k3t9A/sBIeoC/T3I+7SfetgG2nGL+caf7Off+mP2JwJOSPBTYtKpO7+nHA0+ZZls+Pst2AHwNOC7JHwDrTjONJK2U9RY7A5I0Rz8ZGf4ZcL8u426q3+V8EbAEeHxV3ZVkOfALKzHdOOucycS2/Ixp2uOqekWSJwD7AeckeXxV3TjH9UjSjLxCKGlt9DXgBX34RSPpDwWu60HebwEP7+m3ARuPMd1k6wDP7cO/C3y1qm4Bbk7y5J7+Elp38rjuk5ckj6yqM6vq9cD1wHZzWJYkjcUrhJLWRq8CPpTkr4BPjqR/EPhUkguAs4FvA1TVjUm+luRC4DPAW6aabgo/AnZL8jfAdcDze/rBwHuTPAS4DDhkDnn/FPAf/UGWP6Y9YLIjrRv7C8C35rAsSRpLqubawyFJgvaUcVVttNj5kKSVZZexJEnSwHmFUJIkaeC8QihJkjRwBoSSJEkDZ0AoSZI0cAaEkiRJA2dAKEmSNHAGhJIkSQP3/wOHXATG1obzCgAAAABJRU5ErkJggg==\n",
       "text/plain": [
        "<Figure size 720x360 with 1 Axes>"
       ]
      },
      "metadata": {
       "needs_background": "light"
      },
      "output_type": "display_data"
     }
    ],
    "source": [
-    "# Functon to take a slice from a dictionnary - head equivalent\n",
-    "def take(n, iterable):\n",
-    "    \"Return first n items of the iterable as a list\"\n",
-    "    return list(islice(iterable, n))\n",
-    "\n",
-    "# display a slice of it\n",
-    "list_tot_points = []\n",
-    "for key in d:\n",
-    "    distrib = d[key]\n",
-    "    list_tot_points.append(np.sum(distrib))\n",
-    "    \n",
-    "tot_per_key = np.array(list_tot_points)\n",
-    "binwidth = 100\n",
-    "n_keys_less_than_binwidth = np.sum(np.array(tot_per_key < binwidth))\n",
-    "perc_key_to_recover = round(100 * ( n_keys_less_than_binwidth / len(tot_per_key) ), 2)\n",
-    "plt.figure(figsize = (10,5))\n",
-    "plt.hist(tot_per_key, bins = range(min(tot_per_key), max(tot_per_key) + binwidth, binwidth))\n",
-    "plt.title(\"Total number of data points per trip_id / stop_id key. N keys with less than {0} points: {1} ({2}%)\"\\\n",
-    "         .format(binwidth, n_keys_less_than_binwidth, perc_key_to_recover))\n",
-    "plt.show()"
+    "plot_data_points_hist(d_real)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "First we generate a dictionnary with cumulative probability based on frequency of delays, for each keys in our reference dictionnary."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 73,
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def cumul_distri_probas_dict(dico):\n",
+    "    list_tot_points = []\n",
+    "    for key in dico:\n",
+    "        distrib = dico[key]\n",
+    "\n",
+    "        # get total number of elements \n",
+    "        N = np.sum(distrib)\n",
+    "\n",
+    "        # make cumulative distribution probabilities\n",
+    "        cdf_distrib = np.empty((len(distrib)), dtype=float)\n",
+    "        save_x = 0\n",
+    "        for x in range(len(distrib)):\n",
+    "            cdf_distrib[x] = float(distrib[x])/float(N) + float(save_x)/float(N)\n",
+    "            save_x += distrib[x]\n",
+    "\n",
+    "        dico[key] = cdf_distrib\n",
+    "    return dico"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
        "[('1286.TA.26-32-j19-1.12.H__8591182',\n",
        "  array([0.        , 0.64333333, 0.81333333, 0.90333333, 0.95555556,\n",
        "         0.96888889, 0.98444444, 0.99611111, 0.99777778, 0.99888889,\n",
        "         0.99888889, 0.99944444, 0.99944444, 0.99944444, 0.99944444,\n",
        "         0.99944444, 1.        , 1.        , 1.        , 1.        ,\n",
        "         1.        , 1.        , 1.        , 1.        , 1.        ,\n",
        "         1.        , 1.        , 1.        , 1.        , 1.        ,\n",
        "         1.        , 1.        ])),\n",
        " ('1286.TA.26-32-j19-1.12.H__8591184',\n",
        "  array([5.56483027e-04, 4.24596550e-01, 7.31775181e-01, 8.94268225e-01,\n",
        "         9.59933222e-01, 9.86644407e-01, 9.93878687e-01, 9.98330551e-01,\n",
        "         9.98330551e-01, 9.98887034e-01, 9.99443517e-01, 9.99443517e-01,\n",
        "         9.99443517e-01, 9.99443517e-01, 9.99443517e-01, 9.99443517e-01,\n",
        "         9.99443517e-01, 9.99443517e-01, 9.99443517e-01, 9.99443517e-01,\n",
        "         9.99443517e-01, 1.00000000e+00, 1.00000000e+00, 1.00000000e+00,\n",
        "         1.00000000e+00, 1.00000000e+00, 1.00000000e+00, 1.00000000e+00,\n",
        "         1.00000000e+00, 1.00000000e+00, 1.00000000e+00, 1.00000000e+00])),\n",
        " ('1286.TA.26-32-j19-1.12.H__8591195',\n",
        "  array([0.        , 0.60166667, 0.84833333, 0.92777778, 0.96333333,\n",
        "         0.98277778, 0.99166667, 0.99666667, 0.99833333, 0.99888889,\n",
        "         0.99888889, 0.99888889, 0.99888889, 0.99888889, 0.99888889,\n",
        "         0.99944444, 0.99944444, 0.99944444, 0.99944444, 0.99944444,\n",
        "         1.        , 1.        , 1.        , 1.        , 1.        ,\n",
        "         1.        , 1.        , 1.        , 1.        , 1.        ,\n",
        "         1.        , 1.        ]))]"
       ]
      },
-     "execution_count": 73,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "list_tot_points = []\n",
-    "for key in d:\n",
-    "    distrib = d[key]\n",
-    "    \n",
-    "    # get total number of elements \n",
-    "    N = np.sum(distrib)\n",
-    "        \n",
-    "    # make cumulative distribution probabilities\n",
-    "    cdf_distrib = np.empty((len(distrib)), dtype=float)\n",
-    "    save_x = 0\n",
-    "    for x in range(len(distrib)):\n",
-    "        cdf_distrib[x] = float(distrib[x])/float(N) + float(save_x)/float(N)\n",
-    "        save_x += distrib[x]\n",
-    "        \n",
-    "    d[key] = cdf_distrib\n",
-    "\n",
+    "d_all_cdp = cumul_distri_probas_dict(d_all)\n",
+    "take(3, d_all_cdp.items())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[('10.TA.1-11-B-j19-1.1.R__8590314',\n",
+       "  array([0.   , 0.25 , 0.5  , 0.625, 1.   , 1.   , 1.   , 1.   , 1.   ,\n",
+       "         1.   , 1.   , 1.   , 1.   , 1.   , 1.   , 1.   , 1.   , 1.   ,\n",
+       "         1.   , 1.   , 1.   , 1.   , 1.   , 1.   , 1.   , 1.   , 1.   ,\n",
+       "         1.   , 1.   , 1.   , 1.   , 1.   ])),\n",
+       " ('10.TA.1-11-B-j19-1.1.R__8590317',\n",
+       "  array([0. , 0.3, 0.5, 0.7, 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,\n",
+       "         1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,\n",
+       "         1. , 1. , 1. , 1. , 1. , 1. ])),\n",
+       " ('10.TA.1-11-B-j19-1.1.R__8594304',\n",
+       "  array([0. , 0. , 0.5, 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,\n",
+       "         1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,\n",
+       "         1. , 1. , 1. , 1. , 1. , 1. ]))]"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "d_real_cdp = cumul_distri_probas_dict(d_real)\n",
+    "take(3, d_real_cdp.items())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# write dictionnary \n",
+    "with gzip.open(\"../data/distributions_cumulative_real.pkl.gz\", \"wb\") as output_file:\n",
+    "    pickle.dump(d_real_cdp, output_file)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'd' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-11-a1ea2e5960fe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# write dictionnary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mgzip\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"../data/distributions_cumulative.pickle\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"wb\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0moutput_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m: name 'd' is not defined"
+     ]
+    }
+   ],
+   "source": [
     "# write dictionnary \n",
     "with gzip.open(\"../data/distributions_cumulative.pickle\", \"wb\") as output_file:\n",
-    "    pickle.dump(d, output_file)\n",
-    "    \n",
-    "# display a slice of it\n",
-    "take(3, d.items())"
+    "    pickle.dump(d, output_file)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Recover data with few/missing points \n",
+    "### Construct recovery tables \n",
     "\n",
-    "First approach is to simple sum up similar distribution to get a new distribution we can use. For that, we need to have transport type, time (rounded to hour) and stop_id which are valid. We then make all combination of these tree parameters and get the associate distributions"
+    "First approach is to simple sum up similar distribution to get a new distribution we can use. For that, we need to have transport type (`route_desc`), `time` (rounded to hour) and `stop_id` which are valid. We then make all combination of these tree parameters and get the associate distributions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>route_int</th>\n",
+       "      <th>trip_int</th>\n",
+       "      <th>stop_int</th>\n",
+       "      <th>stop_sequence</th>\n",
+       "      <th>arrival_time</th>\n",
+       "      <th>departure_time</th>\n",
+       "      <th>route_id</th>\n",
+       "      <th>trip_id</th>\n",
+       "      <th>stop_id</th>\n",
+       "      <th>route_desc</th>\n",
+       "      <th>stop_id_raw</th>\n",
+       "      <th>sequence_shift_1</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>NaT</td>\n",
+       "      <td>2020-05-21 07:18:00</td>\n",
+       "      <td>30-57-Y-j19-1</td>\n",
+       "      <td>4.TA.30-57-Y-j19-1.1.H</td>\n",
+       "      <td>8502208</td>\n",
+       "      <td>Bus</td>\n",
+       "      <td>8502208</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2020-05-21 07:23:00</td>\n",
+       "      <td>2020-05-21 07:23:00</td>\n",
+       "      <td>30-57-Y-j19-1</td>\n",
+       "      <td>4.TA.30-57-Y-j19-1.1.H</td>\n",
+       "      <td>8502209</td>\n",
+       "      <td>Bus</td>\n",
+       "      <td>8502209</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>3</td>\n",
+       "      <td>2020-05-21 07:29:00</td>\n",
+       "      <td>NaT</td>\n",
+       "      <td>30-57-Y-j19-1</td>\n",
+       "      <td>4.TA.30-57-Y-j19-1.1.H</td>\n",
+       "      <td>8503202</td>\n",
+       "      <td>Bus</td>\n",
+       "      <td>8503202</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>NaT</td>\n",
+       "      <td>2020-05-21 07:48:00</td>\n",
+       "      <td>30-57-Y-j19-1</td>\n",
+       "      <td>5.TA.30-57-Y-j19-1.1.H</td>\n",
+       "      <td>8502208</td>\n",
+       "      <td>Bus</td>\n",
+       "      <td>8502208</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2020-05-21 07:53:00</td>\n",
+       "      <td>2020-05-21 07:53:00</td>\n",
+       "      <td>30-57-Y-j19-1</td>\n",
+       "      <td>5.TA.30-57-Y-j19-1.1.H</td>\n",
+       "      <td>8502209</td>\n",
+       "      <td>Bus</td>\n",
+       "      <td>8502209</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   route_int  trip_int  stop_int  stop_sequence        arrival_time  \\\n",
+       "0          0         0         0              1                 NaT   \n",
+       "1          0         0         1              2 2020-05-21 07:23:00   \n",
+       "2          0         0         2              3 2020-05-21 07:29:00   \n",
+       "3          0         1         0              1                 NaT   \n",
+       "4          0         1         1              2 2020-05-21 07:53:00   \n",
+       "\n",
+       "       departure_time       route_id                 trip_id  stop_id  \\\n",
+       "0 2020-05-21 07:18:00  30-57-Y-j19-1  4.TA.30-57-Y-j19-1.1.H  8502208   \n",
+       "1 2020-05-21 07:23:00  30-57-Y-j19-1  4.TA.30-57-Y-j19-1.1.H  8502209   \n",
+       "2                 NaT  30-57-Y-j19-1  4.TA.30-57-Y-j19-1.1.H  8503202   \n",
+       "3 2020-05-21 07:48:00  30-57-Y-j19-1  5.TA.30-57-Y-j19-1.1.H  8502208   \n",
+       "4 2020-05-21 07:53:00  30-57-Y-j19-1  5.TA.30-57-Y-j19-1.1.H  8502209   \n",
+       "\n",
+       "  route_desc stop_id_raw  sequence_shift_1  \n",
+       "0        Bus     8502208                 2  \n",
+       "1        Bus     8502209                 3  \n",
+       "2        Bus     8503202                 1  \n",
+       "3        Bus     8502208                 2  \n",
+       "4        Bus     8502209                 3  "
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "with open(\"../data/stop_times_df.pkl\", \"rb\") as input_file:\n",
+    "    stoptimes = pickle.load(input_file)\n",
+    "    \n",
+    "stoptimes.head()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 42,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "0.0%, 4.03%, 8.07%, 12.1%, 16.13%, 20.17%, 24.2%, 28.23%, 32.27%, 36.3%, 40.34%, 44.37%, 48.4%, 52.44%, 56.47%, 60.5%, 64.54%, 68.57%, 72.6%, 76.64%, 80.67%, 84.7%, 88.74%, 92.77%, 96.81%, "
+      "0.0%, 4.07%, 8.14%, 12.21%, 16.28%, 20.35%, 24.42%, 28.49%, 32.55%, 36.62%, 40.69%, 44.76%, 48.83%, 52.9%, 56.97%, 61.04%, 65.11%, 69.18%, 73.25%, 77.32%, 81.39%, 85.46%, 89.53%, 93.6%, 97.66%, "
      ]
     }
    ],
    "source": [
-    "with open(\"../data/stop_times_df.pkl\", \"rb\") as input_file:\n",
-    "    stoptimes = pickle.load(input_file)\n",
-    "    \n",
     "# Set same stoptimes index as distribution dict \n",
+    "stoptimes['stop_id'] = stoptimes['stop_id'].astype(str).str[0:7]\n",
     "stoptimes['key'] = stoptimes['trip_id'] + '__' + stoptimes['stop_id']\n",
     "stoptimes = stoptimes.set_index('key')\n",
     "\n",
-    "stoptimes = stoptimes[['trip_id','stop_id','arrival_time', 'departure_time']]\n",
+    "stoptimes = stoptimes[['trip_id','stop_id', 'route_desc', 'arrival_time', 'departure_time']]\n",
     "\n",
     "list_hours = []\n",
     "size_stop_times = stoptimes.shape[0]\n",
     "for x in range(size_stop_times):\n",
     "    if (x % 10000) == 0 :\n",
     "        print('{}%'.format(round(100*x/size_stop_times,2)), end = ', ')\n",
     "    \n",
     "    arr_time_hour = pd.to_datetime(stoptimes.iloc[x,:]['arrival_time']).hour\n",
     "    if math.isnan(arr_time_hour): # if arrival is NaT, use departure time\n",
     "        arr_time_hour = pd.to_datetime(stoptimes.iloc[x,:]['departure_time']).hour\n",
-    "    list_hours.append(arr_time_hour)\n",
+    "    list_hours.append(int(arr_time_hour))\n",
     "    \n",
     "stoptimes['hour'] = list_hours\n",
     "stoptimes = stoptimes.drop(columns=['trip_id', 'arrival_time', 'departure_time'])\n",
-    "              \n",
+    "\n",
     "# Write this pickle to avoid re-running this above code all the time\n",
     "with gzip.open(\"../data/stop_times_wHour.pkl\", \"wb\") as output_file:\n",
     "    pickle.dump(stoptimes, output_file)         \n",
     "              "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 15,
    "metadata": {},
-   "outputs": [],
-   "source": [
-    "with gzip.open(\"../data/stop_times_wHour.pkl\", \"rb\") as input_file:\n",
-    "    stoptimes = pickle.load(input_file)\n",
-    "    \n",
-    "distrib_df = pd.DataFrame(d).transpose()\n",
-    "stoptimes_df = pd.DataFrame(stoptimes)\n",
-    "stoptimes_df['stop_id'] = stoptimes_df['stop_id'].str[0:7]\n",
-    "join_df = pd.concat([distrib_df, stoptimes_df], join='outer')\n",
-    "list_bins = [x for x in range(31)]\n",
-    "join_df = join_df.groupby(['hour', 'stop_id'])[list_bins].apply(lambda x : x.astype(float).sum())\n",
-    "print(join_df.sum(axis=0))\n",
-    "join_df.head()"
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(17321, 32)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th>0</th>\n",
+       "      <th>1</th>\n",
+       "      <th>2</th>\n",
+       "      <th>3</th>\n",
+       "      <th>4</th>\n",
+       "      <th>5</th>\n",
+       "      <th>6</th>\n",
+       "      <th>7</th>\n",
+       "      <th>8</th>\n",
+       "      <th>9</th>\n",
+       "      <th>...</th>\n",
+       "      <th>22</th>\n",
+       "      <th>23</th>\n",
+       "      <th>24</th>\n",
+       "      <th>25</th>\n",
+       "      <th>26</th>\n",
+       "      <th>27</th>\n",
+       "      <th>28</th>\n",
+       "      <th>29</th>\n",
+       "      <th>30</th>\n",
+       "      <th>31</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>stop_id</th>\n",
+       "      <th>hour</th>\n",
+       "      <th>route_desc</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th rowspan=\"4\" valign=\"top\">8500926</th>\n",
+       "      <th>8.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>...</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>...</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>...</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>...</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>4 rows × 32 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                         0   1   2   3   4   5   6   7   8   9   ...  22  23  \\\n",
+       "stop_id hour route_desc                                          ...           \n",
+       "8500926 8.0  Bus          0   2   3   3   4   4   4   4   4   4  ...   4   4   \n",
+       "        9.0  Bus          0   1   2   2   2   2   2   2   2   2  ...   2   2   \n",
+       "        10.0 Bus          0   0   1   1   1   1   1   2   2   2  ...   2   2   \n",
+       "        11.0 Bus          0   0   1   1   1   2   2   2   2   2  ...   2   2   \n",
+       "\n",
+       "                         24  25  26  27  28  29  30  31  \n",
+       "stop_id hour route_desc                                  \n",
+       "8500926 8.0  Bus          4   4   4   4   4   4   4   4  \n",
+       "        9.0  Bus          2   2   2   2   2   2   2   3  \n",
+       "        10.0 Bus          2   2   2   2   2   2   2   2  \n",
+       "        11.0 Bus          2   2   2   2   2   2   2   2  \n",
+       "\n",
+       "[4 rows x 32 columns]"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "with gzip.open(\"../data/stop_times_wHour.pkl\", \"rb\") as input_file:\n",
+    "    stoptimes = pickle.load(input_file)\n",
+    "    \n",
+    "distrib_df = pd.DataFrame(d_all).transpose()\n",
+    "distrib_to_rm = np.array(distrib_df.iloc[:,range(11)].sum(axis=1) == 11) # missing trips\n",
+    "distrib_df = distrib_df.iloc[~distrib_to_rm,:]\n",
+    "\n",
+    "stoptimes_df = pd.DataFrame(stoptimes)\n",
+    "\n",
+    "recovery_df = distrib_df.join(stoptimes_df)\n",
+    "list_bins = [x for x in range(32)]\n",
+    "\n",
+    "recovery_df = recovery_df.groupby(['stop_id','hour', 'route_desc'])[list_bins].apply(lambda x : x.astype(float).sum())\n",
+    "recovery_df = recovery_df.astype('int')\n",
+    "print(recovery_df.shape)\n",
+    "recovery_df.head(4)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def plot_df_missing(df, max_bin = 10000):\n",
+    "    tot_per_key = np.array(df.sum(axis=1)).astype('int')\n",
+    "    binwidth = 100\n",
+    "    n_keys_less_than_binwidth = np.sum(np.array(tot_per_key < binwidth))\n",
+    "    perc_key_to_recover = round(100 * ( n_keys_less_than_binwidth / len(tot_per_key) ), 2)\n",
+    "    plt.figure(figsize = (10,5))\n",
+    "    plt.hist(tot_per_key, bins = range(min(tot_per_key), max_bin + binwidth, binwidth))\n",
+    "    plt.title(\"Total number of data points per stop_id / hour key. N keys with less than {0} points: {1} ({2}%)\"\\\n",
+    "             .format(binwidth, n_keys_less_than_binwidth, perc_key_to_recover))\n",
+    "    plt.xlabel('n data points')\n",
+    "    plt.ylabel('n keys')\n",
+    "    return plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAn8AAAFNCAYAAABxFAnAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd5xtZX3v8c9XmgWlCDFSwkFFE4xXokQxlqAkgqJirEQENOZyLUmMN4mCURFLgjHW2KKigg2RWFDxKkEFKwoWqsihhSqHXkSl/O4fzzOwz5wpe2DmzJyzPu/Xa16z97PWXutZ/buetdbeqSokSZI0DHdZ7ApIkiRp9TH8SZIkDYjhT5IkaUAMf5IkSQNi+JMkSRoQw58kSdKADCb8JakkD1gC9fhWkr9epHHfLcmXklyT5LNj9L9zkgtXR93ujCR7Jfn6YtdjTZfk1Uk+PEP385L82R0c9mKu9y9I8p3FGPc4knwsyZsWux4zmW0bm+u+YjHXhz7+JT/PV6ckH0jy2sWux2JKskGS05Pcd7Hrcmck+WGSB8/W36KHvyTXj/zdmuTGkfd7TfOZNSKULEHPAu4D3Luqnj2fA17MA2xVfbKqnjhOv0s9CEwlybJ+8rLuQo6nqv6lqu7UATnJmUkeOF91WtP09auSvHJS+YVJdl6kat1pk7expXIyPY7F3uaTvDHJKUluTvL6Kbo/L8n5SW5I8oUkm4502zTJ53u385M8byHqWFUvrqo3jtPvfAbnJK9Ick6Sa5NcnOQdo/u52ebdSH8fmbxO9v3m0UmuSnJpkvfMsg/dDzi+qi7pn398km/2xpLzphjnDkm+3btfOFN4TvKHSb6W5PIkq3y58qQcdH2SW5L8R++2dZIfJLkyydsmfe6rSXacNLh/B94ww3QCSyD8VdWGE3/A/wBPHSn75GLXb6lKM9fltw3wi6q6eSHqpPEtdJBbDEnuD6xTVb9YxDoshfl6JfDKJPdc7IpoSVgOvBL4yuQOvYXmP4G9aSfmvwLeN9LLe4Hf9m57Ae8fp1VnDXIU8LCquhfwh8BDgb8b6T7tvJuQ5DHA/afo9D7gMuC+wA7AnwIvnaEuLwY+PvL+BuAjwD9N0/+ngOOBTSeGneRp0/R7E3AE8KKpOk7KQb8L3AhMXJ07ADgU2BZ4+kTYS/Jc4NyqOnHS4I4CHp/kd6ebUFgC4W86vQn2nf1s4OL+eoMk9wC+CmwxkpK3SPKIJN9PcnWSS3rKX3/McX2rn2F8N8l1Sb6eZLPebZVWxoxc/kry+iSfTfKJ/tlTkjwwyQFJLktyQZLJrVL3T2uavTbJFyed6e2U5Ht9On6WkdaCXs83J/kubSdxvymm5Q96f1cnOW1iZUxyEPA64Ll9nq2yEqZdFv5YP1M6HfjjSd33T3J2n87Tk/zFxDiBDwCP6sO+upfvnuQnfTovmOXMbed+9vTqfnZ0XkZafpNslOSwJCvSzoBfMxF+M+nMPu0M8MVJzurz4b09LE9Xzyf36bkuyUVJ/nGaOr6gryPvSTvb+3mSXSbV8ZC+/l2U5E1J1pn02XckuQJYZV70dfjEPr9+meTtvdPx/f/Vvd6PSnKXPg/O7+vZYUk26sOZaCncL23buWS6aZo0/tcn+cTI+7378K9I8s+zfR7YHTh6hu7bZIptrI/raX19vbqvv38w0m3yGf1tLQ8j682rklwKfHSM6Xxrku/05TXlMkuyftqZ9kNGPvc7SX6VZPNZRnEG8H3g/85Wlynqds+01oZ393X295Mc0+tyZpLn9P7+uK8j64x89hlJftZfT7cuTR7fcUme2V8/us/r3fv7XZL8tL++bRtLMrE+/qyvj88dGd4/9PXxkiQvnMN0/1WSM9L2PV9Lsk0vT99mLuvTckqSP+zdZt1uM802322S5Cv98yeknbxMfO5dafusa5OclOSxI91en+SIvs1d19fbya0vt6mqQ6vqq8B1U3TeC/hSVR1fVdcDrwWe0deDewDPBF5bVddX1XdoB/a9p5mHr09yZJLP9Hr9OMlDR+dFpjg29G5TbVOrLMsk+/U6v7LPzy/18lf1ZXBdX093YQxVdXZVTSyTALcCDxjpPtO8mzjZ+w/gb6fovC1wRFX9uqouBf4fMGVwTvJ7tOPpCSPj/mFVfRw4Z5rqLwM+WVW3VNXZwHemG35VnVlVhwCnTTOsUc+khdZvj0zHN6rqGuBHwP2S3AvYH3j1FOP6NXASsOtMI1my4Q/4Z2AnWmJ/KPAI4DVVdQPwJODikbR8MXAL8ApgM+BRwC7MnPInex7wQuB3gPWBWQ+WI55KO2PYBPgJ8DXavN2S1vz6n5P63wf4K9oZyc3AuwGSbEk7w3kT7WziH4H/mnSw2ZvWPH1P4PzRgSZZD/gS8PU+HX8LfDLJg6rqQOBfgM/0eXbIFNNxIO0M6v60FWffSd3PBh4LbAQcBHwiyX2r6gzaWdP3+7A37v3f0Kd1Y1oweEmSp081A7vfpS2/Lfu4P5jkQb3bf/Tx3o92lrUPbXlN5ym08Pq/gOcAu85Qz0OA/1NV96SdfX5jhuE+ss+HzWjz63O5Pbx/jLY8HwD8EfBE4K8nffYc2ln8m6cY9ruAd/Wz4PvTzhQBHtf/b9zr/X3gBf3v8bR5siHwnknDezywXa/HqzKH+/WSbA+8n7a+bQHcG9hqlo89mRnO0JlmG0u7TPxp4O+BzWkB8ksZ8+SNtt5sSmvZ3m+6ntIC84do68QT+870Y0yxzKrqt8DhwPNHBvGXwLFVtWKMOr0W+PuRdWNWSe4NHAt8t6r+Drg7cAytheF3gD2B9yXZvqp+BFzR6zthb+Cw/nq6dWmy44Cd++s/pa2fjxt5f9zkD1TVRPeH9vXxM/3979K20S1pLRzvTbLJGNO9B+0g9gza8v82bX2gT9/jgAf2YT+nTzeMsd3OsM1Dm58H0fbby1l5m/wR7dizKW3+fzbJXUe6P422fmxMC2STt71xPRj42Uh9z6a19D2w/908qSX9Z0wTMLo9aC1GE/X+QpL1Zjo2TDOcKZdlVX0Q+CTwb31+PrUP42+AP+7LYlfgPGitcpMC9yrSLntfC1xOO9ZPPl7O5BW0S7UnT9HtncCeSe7ej61PogXAqTwEOGeOV8XeCezT5++DaLnjv+fw+ensCxxWt//27qnAnyfZGHg4LUC+EXjnSHCe7AzavJzWUg5/ewFvqKrL+s72IKY54wGoqpOq6gdVdXNVnUdbgf50DuP7aFX9oqpupO0od5jDZ79dVV/rK85naTuwg6vqJtoOYllfcBM+XlWn9iD7WuA5aWfwzweOrqqjq+rWqjoGOJF2UJ3wsao6rU/nTZPqsRMtBBxcVb+tqm8AX6YdtMbxHODNVXVlVV1AD6UTquqzVXVxr9tngLNooXxKVfWtqjql938ybYc+2zJ5bVX9pqqOowWJiXmzJ3BAVV3Xl+/bmGF9oM2Dq6vqf4BvMvPyvAnYPsm9quqqqvrxDP1eRtvoburz4Exg9yT3oS2nv6+qG6rqMuAdvd4TLq6q/+jL7sZp6vGAJJv1M/0fzFCPvYC3V9U5vcXgANqObvSy50G9LqfQWsTGXQ+g3R/65d4i8RvaenrrdD0nuTstbH9rhmFOt409F/hKVR3T1+l/B+4G/MmYdb0VOLCvN1PNV4D1aOvfprRbS341xjI7FPjLJOnv92bly0LTqqqf0oLbq8achi1oQeuzVfWaXvYU4Lyq+mhfZ34C/Bcwcb/uofRw2kPmrrQDPoy/Lh3H7dvk44B/HXk/ZfibwU20ffZNVXU0cD0wXbgY9WLgX6vqjL4P/Rdgh976dxPtRPf3gfR+LhkZ37jb7VQ+X61152ZaoLltH1FVn6iqK/p8fxuwwaRp+U7fT99CWydmPNDOYEPgmkll19CmeUPg2mm6Teekqjqyb0dvB+5KOy7M9dgwl2V5C23+bJ9kvao6r4dYquo7kwL3KqrqU/0k5YG0VtpfztT/hCRbA/+HdkVrKsfTgvK1wIW0Y+kXpul3Y6ZpXZzBl2n7yRuBnwOH9JOyO6yv839K27Yn/Cut0eU42qXs9WknsF9K8qkkxyf5m0mDuo42TdNayuFvC1Zu2Tq/l00p7VLrl9Nu7LyWtgPZbLr+p3DpyOtf0TaUcY2urDcCl/edwsR7Jg3vgpHX59MOTJvRWi6e3Zvlr+5nTI+htRBO9dnJtgAuqKrRg/T5tLO3cWwxRd1uk2SfJD8dqdsfMsM8TvLItEtYK5JcQ9vJz7RMruqBeHT8W/TPrMeq68NM0zWX5flMWgg4P+0y2KNm6PeikTOy0Tpu0+t4ycj8+U/aWfaEmZYdtDPsBwI/T/KjJE+Zod+pto91aa2KU41vxu1nmuHf9vm+XK6Yvnd2Ab7Xg+J0plsmK01LX38vYPz1dkW1Sx0zeQCtVeSgaq16MMsyq6oTej13TvL7fRhHjVknaAell/SQOZvdaYH3AyNl2wCPnLQ/2IvWKgPwCeCp/fLgc2gnoRPBaNx16fvAA3sdd6C1HG6ddkn+Edx+y8E4rpjUcjLufnQb4F0j03gl7RLglj2kvId279tlST7YL3nB3LbbqUy7j0jyj2mXoa/pddqIlfddkz9719yx+02vB+41qexetIP3TN2mM7rN3koLPVsw92PD2MuyqpbTWu1fT1tGhyeZy75mYjhn0Vq13jdbv907aQF1cngm7Zag/wd8DrgHbdltArxlmmFdxcyhevLwN+3DfwMtYG8N7JpkLlcbp7I37cTi3ImC3hjz3Kp6KK1Ff+Iy9/60VsE/A16ckVtl+rTM2OK6lMPfxbSdwoTf62UAqzwtQ7tE9XNgu34W8WraDuTOuoF2+QWA3go12z0/s9l65PXv0c6yLqdtuB+vqo1H/u5RVQeP9D/VtE+4mLbjHl2uvwdcNGa9LpmibsBtZyQfojXv37ufzZ3K7fN4qnp9inaw3LqqNqId2GZaJpv0A9no+C+mzZubWHV9GHe6Rq1Sz6r6UVXtQTvof4HpL5EBbDnSEjRaxwuA3wCbjSy7e1XV6CWamZYdVXVWVf1lr8dbgCP7/Jjqc1NtHzez8onI5GV5MeNbaV3oLXv3nqH/JzPz/X4zWWla+vzdmtuX768Y2Qa5PfxMmHG+dmfQLjl/deRS1zjLbKJ1bW/gyDFC5u2Vqvo57eAzzv2SH6IdTI4e2QYuAI6btD/YsKpe0od/ES28PYNJrZIzrEuT6/gr2v1BLwdO7cH4e7T7Fc+uqsvHnd474QLa5dvR6bxbVX2v1/HdVfVwYHtaoP2nXj7udjvO+nGbtPv7XkkL1Jv0fd01zM/xZLLTGGk1THI/WivaL/rfukm2G+n/ocx839joNnsX2q0aF3Pnjw2jptqHfqqqHkPbjovpQ9Zs1mXqhzemsgvw1t7gMxHGv5/2RPSmtOl7T78icAXt6seTpxnWycC2cwjw9wNuqarDeuvwhbSrfNMNf1z7sHKr32T7AT+oqlNpl6pP7NvsKf39hD9g5HaCqSzl8Pdp4DVJNu9noa+jnelCO8DdO/0G9+6etObd6/tZ+kvmqR6/oJ3V7d7vm3gNbeO8M56fZPt+QH0D7aByC7efye+adtP5XdNuvp3tXqsJEy0Vr+z3IexMux/x8DE/fwRwQJJN+jhHb6KdCCErANJuAP7Dke6/BLbKyvdp3RO4sqp+neQRtHu+ZnNQ2s32j6Vd9vpsnzdHAG9OuxF6G9rB6RMzDWgaK9Wzj2uvJBv1SyXXMsPlTdqB5u/6/H02bSM7ure4fB14W5J7pd1fdv8kY996kOT5STbvZ+cTZ2230ub5raz8gM+ngVck2TbJhtx+P+fo2fpr0+53eTAt+HyG8R0JPCXtnp31aevpTPuLJzHz/X4zOYJ26XyXvo39Ay2Ufa93/ynwvL5N7Mbcbue4TVV9mnZS+N9J7j/mMvsE8Be0AHjYqkOd1UG0eT/jJZjub2i3EXwpyd1ol5UemPbgzXr9748nneEfRgsqD6EFTWDGdWkqx/VxT1zi/dak91P5JVM8cHYHfYC233kw3Pbg1LP76z9Ou4KwHu1E/NfArXPcbqfaN83knrQTqRW08PU6Vm2BG1tfbnelbT/r9v36xIM6n6Tt8x/bw/kbgM9Vu73lBtoyfUOSeyR5NK31eqZbDx6e9uDPurTWuN8AP+DOHxtGrbTskzwoyROSbEBbPjcy8z70Nkn+Osnv9Nfb025fOXak+0zz7oG0MLwDt1+yfyrtcv7lwLm0lvd102672pcW8lbRw9tyRm5j6vuDu9KuDqSPe2Id+kUve17v73dpt69MOfw0d6VdsqUPa4NJ/fwJrSV2yu/g7fPpZdz+sOC5tKd6NwR2pD+Y0sfzcNptJ9NayuHvTbRr9CfTUu2Pe9nEGfWngXPSLhVsQbt5/Hm0JvEPMbcD3bR6k/JLgQ/TzpJuoDWl3xkfp91ofimtyfjv+rguoG3cr6bteC6gneWOtZz6GcBTaQfiy2nN5/v0+TWOg2iXAs6lHRRHWxJOp91n933axv8Q4Lsjn/0G7Yz00iQTrQUvpe24rqOF95la1KDNj6toZ6mfBF48Uve/pc37c2hPVX2K9hj+XE1Vz72B89JuF3gx7dLadE6gPURxOe0G8Wf1s0poZ23rA6f36TiSlS/Zz2Y34LQk19Oa9/esqht768ybge/29X0n2rR/nHZZ7lzaTnfyE2/H0XZoxwL/XlVjfxF2VZ1G29F8itYKeBXTrPdpT19eX+3+yjmrqjNp4eo/aPP1qbT78iYuz768l01c9pzuvp1xxnUo7QD7jSTLmGWZ9W3yx7QTn28DpD0pOdM6Mjq+c2nLaZVWtyn6LdqZ/YXAF2mt3U+k3YN4MW37eAsrn3x+ntba8vm+nkyYcl2aZtTH0QLP8dO8n8rrgUP7+vic2aZtJlX1edp0Hd63wVNp+zBooetDtGVzPu3Wg7f2buNut1Nt8zP5Gq0V9hd9nL9m9ls2ZvIhWiD6S1or8I297hPb2Ytp+7vLaPN99NLhS2m3A1xGO+a9pH9mOl+khZCr+jie0e/bu7PHhlGH0O7vuzrJF2jr48F9uJfSTpAPgNaK2tfB6TwaOCXJDbQrB0ez8hOsM827y6rq0om/3v/lI+v5M2jbwQrafvAm2gMi05n4yp0Jj+vjO5rWingj7bhIVV3bh/8K2rz+KW29nXhi+vfSnoaeuHq2Tf/8xLK7kXaiN2pfevCfpn7/TrvMPTE//xV4Am3d/FLd/pUvTwW+Ve1B2Gll5duXpMXRz0Q/UVXjtnKudkleQHsS9DGLXZeZ9FBzLrBerYbvdEz7QuPNquqVs/a8BkryEdrDOq+ZtedFkORs2mXT+XjSUGuotK/SekBVPX+2frWq3hL3E2CXkXtn1zhJTgBe1C8NT2spfCGqpDXbebSvkVjr9CD9DNrXwCw5ad/RV8z89USSZlHtYbXtF7sed1ZVPXKc/pbyZV9J8yzt54Am/5TQ9UlW+bLQcVXVEdW+T22tkuSNtEs5bx19+m6pSPIt2oNuL6uVn+KUpBl52VeSJGlAbPmTJEkaEMOfJEnSgKyVD3xsttlmtWzZssWuhiRJ0qxOOumky6vqzv6AxNjWyvC3bNkyTjzxxNl7lCRJWmRJzp+9r/njZV9JkqQBMfxJkiQNiOFPkiRpQAx/kiRJA2L4kyRJGhDDnyRJ0oAY/iRJkgbE8CdJkjQghj9JkqQBMfxJkiQNiOFPkiRpQNbK3/ZdTMv2/8oqZecdvPsi1ESSJGlVtvxJkiQNiOFPkiRpQAx/kiRJA2L4kyRJGhDDnyRJ0oAY/iRJkgbE8CdJkjQghj9JkqQBMfxJkiQNiOFPkiRpQAx/kiRJA2L4kyRJGhDDnyRJ0oAY/iRJkgbE8CdJkjQghj9JkqQBMfxJkiQNiOFPkiRpQBY8/CVZJ8lPkny5v982yQlJlif5TJL1e/kG/f3y3n3ZyDAO6OVnJtl1oessSZK0tlodLX8vB84Yef8W4B1V9QDgKuBFvfxFwFW9/B29P5JsD+wJPBjYDXhfknVWQ70lSZLWOgsa/pJsBewOfLi/D/AE4Mjey6HA0/vrPfp7evddev97AIdX1W+q6lxgOfCIhay3JEnS2mqhW/7eCbwSuLW/vzdwdVXd3N9fCGzZX28JXADQu1/T+7+tfIrPSJIkaQ4WLPwleQpwWVWdtFDjmDS+/ZKcmOTEFStWrI5RSpIkrXEWsuXv0cDTkpwHHE673PsuYOMk6/Z+tgIu6q8vArYG6N03Aq4YLZ/iM7epqg9W1Y5VtePmm28+/1MjSZK0Fliw8FdVB1TVVlW1jPbAxjeqai/gm8Czem/7Al/sr4/q7+ndv1FV1cv37E8DbwtsB/xwoeotSZK0Nlt39l7m3auAw5O8CfgJcEgvPwT4eJLlwJW0wEhVnZbkCOB04GbgZVV1y+qvtiRJ0ppvtYS/qvoW8K3++hymeFq3qn4NPHuaz78ZePPC1VCSJGkY/IUPSZKkATH8SZIkDYjhT5IkaUAMf5IkSQNi+JMkSRoQw58kSdKAGP4kSZIGxPAnSZI0IIY/SZKkATH8SZIkDYjhT5IkaUAMf5IkSQNi+JMkSRoQw58kSdKAGP4kSZIGxPAnSZI0IIY/SZKkATH8SZIkDYjhT5IkaUAMf5IkSQNi+JMkSRoQw58kSdKAGP4kSZIGxPAnSZI0IIY/SZKkATH8SZIkDYjhT5IkaUAMf5IkSQNi+JMkSRoQw58kSdKAGP4kSZIGxPAnSZI0IIY/SZKkATH8SZIkDYjhT5IkaUAMf5IkSQNi+JMkSRoQw58kSdKAGP4kSZIGxPAnSZI0IIY/SZKkATH8SZIkDYjhT5IkaUDWXewKrMmW7f+Vxa6CJEnSnNjyJ0mSNCCGP0mSpAEx/EmSJA2I4U+SJGlADH+SJEkDYviTJEkakAULf0numuSHSX6W5LQkB/XybZOckGR5ks8kWb+Xb9DfL+/dl40M64BefmaSXReqzpIkSWu7hWz5+w3whKp6KLADsFuSnYC3AO+oqgcAVwEv6v2/CLiql7+j90eS7YE9gQcDuwHvS7LOAtZbkiRprbVg4a+a6/vb9fpfAU8AjuzlhwJP76/36O/p3XdJkl5+eFX9pqrOBZYDj1ioekuSJK3NFvSevyTrJPkpcBlwDHA2cHVV3dx7uRDYsr/eErgAoHe/Brj3aPkUn5EkSdIcLGj4q6pbqmoHYCtaa93vL9S4kuyX5MQkJ65YsWKhRiNJkrRGWy1P+1bV1cA3gUcBGyeZ+E3hrYCL+uuLgK0BeveNgCtGy6f4zOg4PlhVO1bVjptvvvmCTIckSdKabt3Ze7ljkmwO3FRVVye5G/DntIc4vgk8Czgc2Bf4Yv/IUf3993v3b1RVJTkK+FSStwNbANsBP1yoei+EZft/ZZWy8w7efRFqIkmShm7Bwh9wX+DQ/mTuXYAjqurLSU4HDk/yJuAnwCG9/0OAjydZDlxJe8KXqjotyRHA6cDNwMuq6pYFrLckSdJaa8HCX1WdDPzRFOXnMMXTulX1a+DZ0wzrzcCb57uOkiRJQ+MvfEiSJA2I4U+SJGlADH+SJEkDYviTJEkaEMOfJEnSgBj+JEmSBsTwJ0mSNCCGP0mSpAEx/EmSJA2I4U+SJGlADH+SJEkDYviTJEkaEMOfJEnSgBj+JEmSBsTwJ0mSNCCGP0mSpAEx/EmSJA2I4U+SJGlADH+SJEkDYviTJEkaEMOfJEnSgMwa/pI8Osk9+uvnJ3l7km0WvmqSJEmab+O0/L0f+FWShwL/AJwNHLagtZIkSdKCGCf83VxVBewBvKeq3gvcc2GrJUmSpIWw7hj9XJfkAOD5wOOS3AVYb2GrJUmSpIUwTsvfc4HfAC+qqkuBrYC3LmitJEmStCDGafl7NvDRqroKoKr+B+/5kyRJWiON0/J3H+BHSY5IsluSLHSlJEmStDBmDX9V9RpgO+AQ4AXAWUn+Jcn9F7hukiRJmmdjfclzf9r30v53M7AJcGSSf1vAukmSJGmezXrPX5KXA/sAlwMfBv6pqm7qT/2eBbxyYasoSZKk+TLOAx+bAs+oqvNHC6vq1iRPWZhqSZIkaSGMc8/fgcDWSV4IkGTzJNv2bmcscP0kSZI0j8b5bd8DgVcBB/Si9YBPLGSlJEmStDDGeeDjL4CnATcAVNXF+PNukiRJa6Rxwt9v+9O+BZDkHgtbJUmSJC2UccLfEUn+E9g4yf8G/pv21K8kSZLWMOM87fs24M+Aa4EHAa8Djl/ISkmSJGlhjBP+DqmqvwKOAUiyIXA0sMtCVkySJEnzb5zLvhcleR9Akk2Ar+PTvpIkSWukcb7n77XA9Uk+QAt+b6uqjy54zSRJkjTvpr3sm+QZI29PAF4L/BCoJM+oqs8tdOUkSZI0v2a65++pk97/hPYFz0+lfe2L4U+SJGkNM234q6oXrs6KCJbt/5VVys47ePdFqIkkSVpbjfPAhyRJktYShj9JkqQBMfxJkiQNyKxf8pxkA+CZwLLR/qvqDQtXLUmSJC2EcX7h44vANcBJwG8WtjqSJElaSOOEv62qarcFr4kkSZIW3Dj3/H0vyUPmOuAkWyf5ZpLTk5yW5OW9fNMkxyQ5q//fpJcnybuTLE9ycpKHjQxr397/WUn2nWtdJEmS1IwT/h4DnJTkzB7KTkly8hifuxn4h6raHtgJeFmS7YH9gWOrajvg2P4e4EnAdv1vP+D90MIicCDwSOARwIETgVGSJElzM85l3yfdkQFX1SXAJf31dUnOALYE9gB27r0dCnwLeFUvP6yqCvhBko2T3Lf3e0xVXQmQ5BhgN+DTd6RekiRJQzZr+Kuq8+/sSJIsA/6I9hvB9+nBEOBS4D799ZbABSMfu7CXTVcuSZKkOVrw7/lLsiHwX8DfV9W1o916K1/N03j2S3JikhNXrFgxH4OUJEla6yxo+EuyHi34fbKqPteLf9kv59L/X9bLLwK2Hvn4Vr1suvKVVNUHq2rHqtpx8803n98JkSRJWkssWPhLEuAQ4IyqevtIp6OAiSd296V9j+BE+T79qd+dgGv65eGvAU9Mskl/0OOJvUySJElzNM4DH3fUo4G9gVOS/LSXvRo4GDgiyYuA84Hn9AhEDfIAAAznSURBVG5HA08GlgO/Al4IUFVXJnkj8KPe3xsmHv6QJEnS3CxY+Kuq7wCZpvMuU/RfwMumGdZHgI/MX+0kSZKGacEf+JAkSdLSYfiTJEkaEMOfJEnSgBj+JEmSBsTwJ0mSNCCGP0mSpAEx/EmSJA2I4U+SJGlADH+SJEkDYviTJEkaEMOfJEnSgBj+JEmSBsTwJ0mSNCCGP0mSpAEx/EmSJA2I4U+SJGlADH+SJEkDYviTJEkaEMOfJEnSgBj+JEmSBsTwJ0mSNCCGP0mSpAEx/EmSJA2I4U+SJGlADH+SJEkDYviTJEkaEMOfJEnSgBj+JEmSBsTwJ0mSNCCGP0mSpAEx/EmSJA2I4U+SJGlADH+SJEkDYviTJEkaEMOfJEnSgBj+JEmSBsTwJ0mSNCCGP0mSpAEx/EmSJA3IuotdgaFatv9XFrsKkiRpgGz5kyRJGhDDnyRJ0oAY/iRJkgbE8CdJkjQghj9JkqQBMfxJkiQNiOFPkiRpQAx/kiRJA2L4kyRJGhDDnyRJ0oAsWPhL8pEklyU5daRs0yTHJDmr/9+klyfJu5MsT3JykoeNfGbf3v9ZSfZdqPpKkiQNwUK2/H0M2G1S2f7AsVW1HXBsfw/wJGC7/rcf8H5oYRE4EHgk8AjgwInAKEmSpLlbsPBXVccDV04q3gM4tL8+FHj6SPlh1fwA2DjJfYFdgWOq6sqqugo4hlUDpSRJksa0uu/5u09VXdJfXwrcp7/eErhgpL8Le9l05ZIkSboDFu2Bj6oqoOZreEn2S3JikhNXrFgxX4OVJElaq6zu8PfLfjmX/v+yXn4RsPVIf1v1sunKV1FVH6yqHatqx80333zeKy5JkrQ2WN3h7yhg4ondfYEvjpTv05/63Qm4pl8e/hrwxCSb9Ac9ntjLJEmSdAesu1ADTvJpYGdgsyQX0p7aPRg4IsmLgPOB5/TejwaeDCwHfgW8EKCqrkzyRuBHvb83VNXkh0gkSZI0pgULf1X1l9N02mWKfgt42TTD+QjwkXmsmiRJ0mD5Cx+SJEkDYviTJEkaEMOfJEnSgBj+JEmSBsTwJ0mSNCCGP0mSpAEx/EmSJA2I4U+SJGlADH+SJEkDYviTJEkaEMOfJEnSgCzYb/tqaVq2/1dWKTvv4N0XoSaSJGkx2PInSZI0IIY/SZKkATH8SZIkDYjhT5IkaUAMf5IkSQNi+JMkSRoQw58kSdKAGP4kSZIGxPAnSZI0IP7CxxLnL3JIkqT5ZMufJEnSgBj+JEmSBsTwJ0mSNCCGP0mSpAHxgY81kA+BSJKkO8qWP0mSpAGx5W8tYWugJEkahy1/kiRJA2L4kyRJGhDDnyRJ0oAY/iRJkgbE8CdJkjQghj9JkqQBMfxJkiQNiOFPkiRpQAx/kiRJA2L4kyRJGhDDnyRJ0oAY/iRJkgZk3cWugBbOsv2/sthVkCRJS4wtf5IkSQNiy5+mbCE87+DdF6EmkiRpodnyJ0mSNCC2/GlK090vaIugJElrNsOfFoSXkiVJWpq87CtJkjQgtvxpTmzRkyRpzWb4053m9wlKkrTmWGMu+ybZLcmZSZYn2X+x6yNJkrQmWiNa/pKsA7wX+HPgQuBHSY6qqtMXt2aaizvTQjjVpWUvQUuSNHdrRPgDHgEsr6pzAJIcDuwBGP4GYtzgOG5/hkRJ0lCtKeFvS+CCkfcXAo9cpLpoLbCYIXG+W0AlSZqLNSX8zSrJfsB+/e31Sc5cDaPdDLh8NYxH45vXZZK3zNeQ5sdSq88cuK0sPS6TpcdlsjStjuWyzQIPfyVrSvi7CNh65P1Wvew2VfVB4IOrs1JJTqyqHVfnODUzl8nS5HJZelwmS4/LZGlaG5fLmvK074+A7ZJsm2R9YE/gqEWukyRJ0hpnjWj5q6qbk/wN8DVgHeAjVXXaIldLkiRpjbNGhD+AqjoaOHqx6zHJar3MrLG4TJYml8vS4zJZelwmS9Nat1xSVYtdB0mSJK0ma8o9f5IkSZoHhr87wJ+aW32SbJ3km0lOT3Jakpf38k2THJPkrP5/k16eJO/uy+bkJA8bGda+vf+zkuy7WNO0tkiyTpKfJPlyf79tkhP6vP9MfziLJBv098t792Ujwzigl5+ZZNfFmZK1R5KNkxyZ5OdJzkjyKLeVxZfkFX3/dWqSTye5q9vL6pXkI0kuS3LqSNm8bRtJHp7klP6ZdyfJ6p3COaoq/+bwR3vg5GzgfsD6wM+A7Re7XmvrH3Bf4GH99T2BXwDbA/8G7N/L9wfe0l8/GfgqEGAn4IRevilwTv+/SX+9yWJP35r8B/xf4FPAl/v7I4A9++sPAC/pr18KfKC/3hP4TH+9fd9+NgC27dvVOos9XWvyH3Ao8Nf99frAxm4ri75MtgTOBe7W3x8BvMDtZbUvh8cBDwNOHSmbt20D+GHvN/2zT1rsaZ7pz5a/ubvtp+aq6rfAxE/NaQFU1SVV9eP++jrgDNrOdA/agY7+/+n99R7AYdX8ANg4yX2BXYFjqurKqroKOAbYbTVOylolyVbA7sCH+/sATwCO7L1MXiYTy+pIYJfe/x7A4VX1m6o6F1hO2750ByTZiHaAOwSgqn5bVVfjtrIUrAvcLcm6wN2BS3B7Wa2q6njgyknF87Jt9G73qqofVEuCh40Ma0ky/M3dVD81t+Ui1WVQ+uWPPwJOAO5TVZf0TpcC9+mvp1s+Lrf59U7glcCt/f29gaur6ub+fnT+3jbve/drev8uk/m1LbAC+Gi/HP/hJPfAbWVRVdVFwL8D/0MLfdcAJ+H2shTM17axZX89uXzJMvxpjZBkQ+C/gL+vqmtHu/UzLR9bX02SPAW4rKpOWuy6aCXr0i5rvb+q/gi4gXYp6zZuK6tfv49sD1o43wK4B7akLjlD2zYMf3M360/NaX4lWY8W/D5ZVZ/rxb/sTe30/5f18umWj8tt/jwaeFqS82i3PTwBeBft0sjEd4eOzt/b5n3vvhFwBS6T+XYhcGFVndDfH0kLg24ri+vPgHOrakVV3QR8jrYNub0svvnaNi7qryeXL1mGv7nzp+ZWo36vyyHAGVX19pFORwETT1rtC3xxpHyf/rTWTsA1vVn/a8ATk2zSz8Sf2Ms0R1V1QFVtVVXLaOv/N6pqL+CbwLN6b5OXycSyelbvv3r5nv3pxm2B7Wg3TesOqKpLgQuSPKgX7QKcjtvKYvsfYKckd+/7s4nl4vay+OZl2+jdrk2yU1/G+4wMa2la7CdO1sQ/2pNAv6A9bfXPi12ftfkPeAytKf5k4Kf978m0e2COBc4C/hvYtPcf4L192ZwC7DgyrL+i3SS9HHjhYk/b2vAH7MztT/vej3YwWg58Ftigl9+1v1/eu99v5PP/3JfVmSzxp+PWhD9gB+DEvr18gfZEotvK4i+Xg4CfA6cCH6c9sev2snqXwadp91zeRGslf9F8bhvAjn35ng28h/4jGkv1z1/4kCRJGhAv+0qSJA2I4U+SJGlADH+SJEkDYviTJEkaEMOfJEnSgBj+JK3Vklw/S/eNk7x0NdTjDUn+bJZ+dk7yJwtdF0nDZviTNHQbAwse/qrqdVX137P0tjNg+JO0oAx/ktYISZYlOSPJh5KcluTrSe42RX/bJvl+klOSvGmkfMMkxyb5ce+2R+90MHD/JD9N8tYZ+ps8nuuTvKPX5dgkm/fyHZL8IMnJST7ffwmAJB9L8qz++rwkB42M4/eTLANeDLyi1+WxSZ6d5NQkP0ty/HzOT0nDZfiTtCbZDnhvVT0YuBp45hT9vAt4f1U9hPaN/hN+DfxFVT0MeDzwtv5TTPsDZ1fVDlX1TzP0N9k9gBN7XY4DDuzlhwGvqqr/Rft1gAOn+CzA5X0c7wf+sarOAz4AvKPX5dvA64Bdq+qhwNNmnTuSNAbDn6Q1yblV9dP++iRg2RT9PJr2U07QfkprQoB/SXIy7aectgTuM8Xnx+3vVuAz/fUngMck2QjYuKqO6+WHAo+bZlo+N8t0AHwX+FiS/w2sM00/kjQn6y52BSRpDn4z8voWYJXLvt1Uv1u5F7A58PCquinJebTfUb2j/Y0zzplMTMstTLMvrqoXJ3kksDtwUpKHV9UVcxyPJK3Elj9Ja5vvAnv213uNlG8EXNYD3eOBbXr5dcA9x+hvsrsAz+qvnwd8p6quAa5K8thevjftkvC4VqpLkvtX1QlV9TpgBbD1HIYlSVOy5U/S2ublwKeSvAr44kj5J4EvJTkFOBH4OUBVXZHku0lOBb4KvGWq/qZwA/CIJK8BLgOe28v3BT6Q5O7AOcAL51D3LwFH9odM/pb28Md2tEvRxwI/m8OwJGlKqZrrlQpJUpLrq2rDxa6HJM2Vl30lSZIGxJY/SZKkAbHlT5IkaUAMf5IkSQNi+JMkSRoQw58kSdKAGP4kSZIGxPAnSZI0IP8fb/CXoziWI40AAAAASUVORK5CYII=\n",
+      "text/plain": [
+       "<Figure size 720x360 with 1 Axes>"
+      ]
+     },
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "plot_df_missing(recovery_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th>0</th>\n",
+       "      <th>1</th>\n",
+       "      <th>2</th>\n",
+       "      <th>3</th>\n",
+       "      <th>4</th>\n",
+       "      <th>5</th>\n",
+       "      <th>6</th>\n",
+       "      <th>7</th>\n",
+       "      <th>8</th>\n",
+       "      <th>9</th>\n",
+       "      <th>...</th>\n",
+       "      <th>22</th>\n",
+       "      <th>23</th>\n",
+       "      <th>24</th>\n",
+       "      <th>25</th>\n",
+       "      <th>26</th>\n",
+       "      <th>27</th>\n",
+       "      <th>28</th>\n",
+       "      <th>29</th>\n",
+       "      <th>30</th>\n",
+       "      <th>31</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>stop_id</th>\n",
+       "      <th>hour</th>\n",
+       "      <th>route_desc</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th rowspan=\"12\" valign=\"top\">8500926</th>\n",
+       "      <th>8.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>...</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>...</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>...</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>...</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>...</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>...</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>...</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>...</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>...</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>...</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>...</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>...</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th rowspan=\"8\" valign=\"top\">8502186</th>\n",
+       "      <th>8.0</th>\n",
+       "      <th>S-Bahn</th>\n",
+       "      <td>0</td>\n",
+       "      <td>3</td>\n",
+       "      <td>6</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>...</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9.0</th>\n",
+       "      <th>S-Bahn</th>\n",
+       "      <td>0</td>\n",
+       "      <td>4</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>...</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10.0</th>\n",
+       "      <th>S-Bahn</th>\n",
+       "      <td>0</td>\n",
+       "      <td>4</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>...</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11.0</th>\n",
+       "      <th>S-Bahn</th>\n",
+       "      <td>0</td>\n",
+       "      <td>4</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>...</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12.0</th>\n",
+       "      <th>S-Bahn</th>\n",
+       "      <td>0</td>\n",
+       "      <td>3</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>...</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13.0</th>\n",
+       "      <th>S-Bahn</th>\n",
+       "      <td>0</td>\n",
+       "      <td>3</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>...</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14.0</th>\n",
+       "      <th>S-Bahn</th>\n",
+       "      <td>0</td>\n",
+       "      <td>3</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>...</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15.0</th>\n",
+       "      <th>S-Bahn</th>\n",
+       "      <td>0</td>\n",
+       "      <td>3</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>...</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>7</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>20 rows × 32 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                         0   1   2   3   4   5   6   7   8   9   ...  22  23  \\\n",
+       "stop_id hour route_desc                                          ...           \n",
+       "8500926 8.0  Bus          0   2   3   3   4   4   4   4   4   4  ...   4   4   \n",
+       "        9.0  Bus          0   1   2   2   2   2   2   2   2   2  ...   2   2   \n",
+       "        10.0 Bus          0   0   1   1   1   1   1   2   2   2  ...   2   2   \n",
+       "        11.0 Bus          0   0   1   1   1   2   2   2   2   2  ...   2   2   \n",
+       "        12.0 Bus          0   0   1   1   1   2   2   2   2   2  ...   2   2   \n",
+       "        13.0 Bus          0   0   1   1   1   2   2   2   2   2  ...   2   2   \n",
+       "        14.0 Bus          0   0   1   1   2   2   2   2   2   2  ...   2   2   \n",
+       "        15.0 Bus          0   0   1   1   2   2   2   2   2   2  ...   2   2   \n",
+       "        16.0 Bus          0   1   3   3   3   3   3   4   4   4  ...   4   4   \n",
+       "        17.0 Bus          0   1   3   3   3   3   3   3   3   3  ...   4   4   \n",
+       "        18.0 Bus          0   2   3   3   3   3   3   3   3   3  ...   4   4   \n",
+       "        19.0 Bus          0   2   2   2   2   3   3   3   3   3  ...   3   3   \n",
+       "8502186 8.0  S-Bahn       0   3   6   7   7   7   7   7   7   7  ...   8   8   \n",
+       "        9.0  S-Bahn       0   4   7   7   7   7   7   7   7   7  ...   8   8   \n",
+       "        10.0 S-Bahn       0   4   7   7   7   7   7   7   7   7  ...   7   7   \n",
+       "        11.0 S-Bahn       0   4   7   7   7   7   7   7   7   7  ...   7   7   \n",
+       "        12.0 S-Bahn       0   3   7   7   7   7   7   7   7   7  ...   7   7   \n",
+       "        13.0 S-Bahn       0   3   7   7   7   7   7   7   7   7  ...   8   8   \n",
+       "        14.0 S-Bahn       0   3   7   7   7   7   7   7   7   7  ...   8   8   \n",
+       "        15.0 S-Bahn       0   3   7   7   7   7   7   7   7   7  ...   7   7   \n",
+       "\n",
+       "                         24  25  26  27  28  29  30  31  \n",
+       "stop_id hour route_desc                                  \n",
+       "8500926 8.0  Bus          4   4   4   4   4   4   4   4  \n",
+       "        9.0  Bus          2   2   2   2   2   2   2   3  \n",
+       "        10.0 Bus          2   2   2   2   2   2   2   2  \n",
+       "        11.0 Bus          2   2   2   2   2   2   2   2  \n",
+       "        12.0 Bus          2   2   2   2   2   2   2   2  \n",
+       "        13.0 Bus          2   2   2   2   2   2   2   2  \n",
+       "        14.0 Bus          2   2   2   2   2   2   2   2  \n",
+       "        15.0 Bus          2   2   2   2   2   2   2   2  \n",
+       "        16.0 Bus          4   4   4   4   4   4   4   4  \n",
+       "        17.0 Bus          4   4   4   4   4   4   4   4  \n",
+       "        18.0 Bus          4   4   4   4   4   4   4   4  \n",
+       "        19.0 Bus          3   3   3   3   3   3   3   3  \n",
+       "8502186 8.0  S-Bahn       8   8   8   8   8   8   8   8  \n",
+       "        9.0  S-Bahn       8   8   8   8   8   8   8   8  \n",
+       "        10.0 S-Bahn       7   7   7   8   8   8   8   8  \n",
+       "        11.0 S-Bahn       7   7   7   7   7   7   7   8  \n",
+       "        12.0 S-Bahn       7   7   7   7   7   7   7   8  \n",
+       "        13.0 S-Bahn       8   8   8   8   8   8   8   8  \n",
+       "        14.0 S-Bahn       8   8   8   8   8   8   8   8  \n",
+       "        15.0 S-Bahn       7   7   7   7   7   7   8   8  \n",
+       "\n",
+       "[20 rows x 32 columns]"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "recovery_df.head(20)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Make second recovery table\n",
+    "\n",
+    "Here only taking combination of `transport_type x hour`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(127, 32)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th>0</th>\n",
+       "      <th>1</th>\n",
+       "      <th>2</th>\n",
+       "      <th>3</th>\n",
+       "      <th>4</th>\n",
+       "      <th>5</th>\n",
+       "      <th>6</th>\n",
+       "      <th>7</th>\n",
+       "      <th>8</th>\n",
+       "      <th>9</th>\n",
+       "      <th>...</th>\n",
+       "      <th>22</th>\n",
+       "      <th>23</th>\n",
+       "      <th>24</th>\n",
+       "      <th>25</th>\n",
+       "      <th>26</th>\n",
+       "      <th>27</th>\n",
+       "      <th>28</th>\n",
+       "      <th>29</th>\n",
+       "      <th>30</th>\n",
+       "      <th>31</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>hour</th>\n",
+       "      <th>route_desc</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th rowspan=\"4\" valign=\"top\">7.0</th>\n",
+       "      <th>Bus</th>\n",
+       "      <td>0</td>\n",
+       "      <td>5</td>\n",
+       "      <td>7</td>\n",
+       "      <td>9</td>\n",
+       "      <td>9</td>\n",
+       "      <td>10</td>\n",
+       "      <td>10</td>\n",
+       "      <td>10</td>\n",
+       "      <td>11</td>\n",
+       "      <td>11</td>\n",
+       "      <td>...</td>\n",
+       "      <td>11</td>\n",
+       "      <td>12</td>\n",
+       "      <td>12</td>\n",
+       "      <td>12</td>\n",
+       "      <td>12</td>\n",
+       "      <td>12</td>\n",
+       "      <td>12</td>\n",
+       "      <td>12</td>\n",
+       "      <td>12</td>\n",
+       "      <td>12</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>InterRegio</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>...</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Intercity</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>...</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>S-Bahn</th>\n",
+       "      <td>0</td>\n",
+       "      <td>3</td>\n",
+       "      <td>5</td>\n",
+       "      <td>6</td>\n",
+       "      <td>7</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>8</td>\n",
+       "      <td>9</td>\n",
+       "      <td>9</td>\n",
+       "      <td>...</td>\n",
+       "      <td>9</td>\n",
+       "      <td>9</td>\n",
+       "      <td>9</td>\n",
+       "      <td>9</td>\n",
+       "      <td>9</td>\n",
+       "      <td>9</td>\n",
+       "      <td>10</td>\n",
+       "      <td>10</td>\n",
+       "      <td>10</td>\n",
+       "      <td>10</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>4 rows × 32 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                 0   1   2   3   4   5   6   7   8   9   ...  22  23  24  25  \\\n",
+       "hour route_desc                                          ...                   \n",
+       "7.0  Bus          0   5   7   9   9  10  10  10  11  11  ...  11  12  12  12   \n",
+       "     InterRegio   0   0   0   0   0   1   1   1   1   1  ...   2   2   2   2   \n",
+       "     Intercity    0   0   0   0   0   1   1   1   1   1  ...   2   2   2   2   \n",
+       "     S-Bahn       0   3   5   6   7   8   8   8   9   9  ...   9   9   9   9   \n",
+       "\n",
+       "                 26  27  28  29  30  31  \n",
+       "hour route_desc                          \n",
+       "7.0  Bus         12  12  12  12  12  12  \n",
+       "     InterRegio   2   2   2   2   2   2  \n",
+       "     Intercity    2   2   2   2   2   2  \n",
+       "     S-Bahn       9   9  10  10  10  10  \n",
+       "\n",
+       "[4 rows x 32 columns]"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "with gzip.open(\"../data/stop_times_wHour.pkl\", \"rb\") as input_file:\n",
+    "    stoptimes = pickle.load(input_file)\n",
+    "    \n",
+    "distrib_df = pd.DataFrame(d_all).transpose()\n",
+    "distrib_to_rm = np.array(distrib_df.iloc[:,range(11)].sum(axis=1) == 11) # missing trips\n",
+    "distrib_df = distrib_df.iloc[~distrib_to_rm,:]\n",
+    "\n",
+    "stoptimes_df = pd.DataFrame(stoptimes)\n",
+    "\n",
+    "recovery_df2 = distrib_df.join(stoptimes_df)\n",
+    "list_bins = [x for x in range(32)]\n",
+    "\n",
+    "recovery_df2 = recovery_df2.groupby(['hour', 'route_desc'])[list_bins].apply(lambda x : x.astype(float).sum())\n",
+    "recovery_df2 = recovery_df2.astype('int')\n",
+    "print(recovery_df2.shape)\n",
+    "recovery_df2.head(4)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Last recovery table \n",
+    "\n",
+    "Takes only transport type distribution"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(11, 32)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>0</th>\n",
+       "      <th>1</th>\n",
+       "      <th>2</th>\n",
+       "      <th>3</th>\n",
+       "      <th>4</th>\n",
+       "      <th>5</th>\n",
+       "      <th>6</th>\n",
+       "      <th>7</th>\n",
+       "      <th>8</th>\n",
+       "      <th>9</th>\n",
+       "      <th>...</th>\n",
+       "      <th>22</th>\n",
+       "      <th>23</th>\n",
+       "      <th>24</th>\n",
+       "      <th>25</th>\n",
+       "      <th>26</th>\n",
+       "      <th>27</th>\n",
+       "      <th>28</th>\n",
+       "      <th>29</th>\n",
+       "      <th>30</th>\n",
+       "      <th>31</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>route_desc</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>Bus</th>\n",
+       "      <td>1572</td>\n",
+       "      <td>59211</td>\n",
+       "      <td>97405</td>\n",
+       "      <td>115766</td>\n",
+       "      <td>124269</td>\n",
+       "      <td>128687</td>\n",
+       "      <td>131397</td>\n",
+       "      <td>133346</td>\n",
+       "      <td>134908</td>\n",
+       "      <td>136278</td>\n",
+       "      <td>...</td>\n",
+       "      <td>137998</td>\n",
+       "      <td>138003</td>\n",
+       "      <td>138007</td>\n",
+       "      <td>138012</td>\n",
+       "      <td>138014</td>\n",
+       "      <td>138016</td>\n",
+       "      <td>138018</td>\n",
+       "      <td>138021</td>\n",
+       "      <td>138023</td>\n",
+       "      <td>138087</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Eurocity</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>...</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>InterRegio</th>\n",
+       "      <td>33</td>\n",
+       "      <td>74</td>\n",
+       "      <td>107</td>\n",
+       "      <td>141</td>\n",
+       "      <td>174</td>\n",
+       "      <td>207</td>\n",
+       "      <td>240</td>\n",
+       "      <td>273</td>\n",
+       "      <td>306</td>\n",
+       "      <td>339</td>\n",
+       "      <td>...</td>\n",
+       "      <td>371</td>\n",
+       "      <td>371</td>\n",
+       "      <td>371</td>\n",
+       "      <td>372</td>\n",
+       "      <td>372</td>\n",
+       "      <td>372</td>\n",
+       "      <td>372</td>\n",
+       "      <td>372</td>\n",
+       "      <td>372</td>\n",
+       "      <td>372</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Intercity</th>\n",
+       "      <td>9</td>\n",
+       "      <td>19</td>\n",
+       "      <td>29</td>\n",
+       "      <td>39</td>\n",
+       "      <td>49</td>\n",
+       "      <td>59</td>\n",
+       "      <td>69</td>\n",
+       "      <td>79</td>\n",
+       "      <td>89</td>\n",
+       "      <td>99</td>\n",
+       "      <td>...</td>\n",
+       "      <td>109</td>\n",
+       "      <td>109</td>\n",
+       "      <td>109</td>\n",
+       "      <td>109</td>\n",
+       "      <td>109</td>\n",
+       "      <td>109</td>\n",
+       "      <td>109</td>\n",
+       "      <td>109</td>\n",
+       "      <td>109</td>\n",
+       "      <td>109</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>4 rows × 32 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "              0      1      2       3       4       5       6       7   \\\n",
+       "route_desc                                                               \n",
+       "Bus         1572  59211  97405  115766  124269  128687  131397  133346   \n",
+       "Eurocity       0      0      0       1       1       1       1       2   \n",
+       "InterRegio    33     74    107     141     174     207     240     273   \n",
+       "Intercity      9     19     29      39      49      59      69      79   \n",
+       "\n",
+       "                8       9   ...      22      23      24      25      26  \\\n",
+       "route_desc                  ...                                           \n",
+       "Bus         134908  136278  ...  137998  138003  138007  138012  138014   \n",
+       "Eurocity         2       2  ...       3       3       3       3       3   \n",
+       "InterRegio     306     339  ...     371     371     371     372     372   \n",
+       "Intercity       89      99  ...     109     109     109     109     109   \n",
+       "\n",
+       "                27      28      29      30      31  \n",
+       "route_desc                                          \n",
+       "Bus         138016  138018  138021  138023  138087  \n",
+       "Eurocity         3       3       3       3       3  \n",
+       "InterRegio     372     372     372     372     372  \n",
+       "Intercity      109     109     109     109     109  \n",
+       "\n",
+       "[4 rows x 32 columns]"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "with gzip.open(\"../data/stop_times_wHour.pkl\", \"rb\") as input_file:\n",
+    "    stoptimes = pickle.load(input_file)\n",
+    "    \n",
+    "distrib_df = pd.DataFrame(d_all).transpose()\n",
+    "distrib_to_rm = np.array(distrib_df.iloc[:,range(11)].sum(axis=1) == 11) # missing trips\n",
+    "distrib_df = distrib_df.iloc[~distrib_to_rm,:]\n",
+    "\n",
+    "stoptimes_df = pd.DataFrame(stoptimes)\n",
+    "\n",
+    "recovery_df3 = distrib_df.join(stoptimes_df)\n",
+    "list_bins = [x for x in range(32)]\n",
+    "\n",
+    "recovery_df3 = recovery_df3.groupby(['route_desc'])[list_bins].apply(lambda x : x.astype(float).sum())\n",
+    "recovery_df3 = recovery_df3.astype('int')\n",
+    "print(recovery_df3.shape)\n",
+    "recovery_df3.head(4)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Reconstruct cumulative distribution probabilities from multiple distributions to recover data with few/missing points \n",
+    "\n",
+    "To recover missing or faulty data, the strategy is the following :\n",
+    "1. If we have more than 100 data points in `real` group, we rely exclusively on it to compute probabilities for a given transfer on a `trip_id x stop_id` \n",
+    "    - `real` group : the delay was calculated with actual arrival time with status `geschaetz` or `real`, meaning it comes from actual measurments.\n",
+    "2. If we do not find enough data within `real` group, we look at distributions in `all` group (contains all delays including `prognose` status) to compute probabilities, if there is more than 100 data points for a given `trip_id x stop_id`.\n",
+    "3. If `all` group still does not have more than 100 data points, we rely on `recovery tables` to estimate delay distributions. The strategy is the following :\n",
+    "    - As we will always know the `stop_id`, the `time` and the `transport_type`, we rely on arrival delays from aggregated values of similar transfer. \n",
+    "        - First, we compute a table of distribution with all possible combination of `stop_id`, `time` (round to hours) and `transport_type`, and aggregate all the counts we have to compute cumulative distribution probabilities. \n",
+    "        - Is there is less than 100 data points in one of these intersections, we use the last possibilities : a table with `transport_type` x `time` aggregate counts.\n",
+    "        - The last values with no match are given the overall average of cumulative distribution probabilities for each `transport_type` with no limit for the minimum number of data points.\n",
+    "\n",
+    "Following this approach, we can find cumulative distribution probabilities for every combination of `trip_id x stop_id` as defined in `stop_times_df`. We will make a table with the same row order so that McRaptor can easily find their indexes. \n",
+    "\n",
+    "In order to do that, we have two dictionnaries of distributions and two recovery dataframes :\n",
+    " - `df_real` : contains delay distribution for each keys in form `trip_id + __ + stop_id` calculated from delays with status `geschaetz` or `real` in sbb datasets.\n",
+    " - `df_all` : contains delay distributions for each keys in form `trip_id + __ + stop_id`. No filter was applied on status (contains `geschaetz`, `real` __and__ `prognose` = evaluated delay).\n",
+    " - `recovery_df` : contains aggregated delay distributions for each combination of `stop_id`, `route_desc` (transport type) and `hour` (time rounded to hour). \n",
+    " - `recovery_df2` : contains aggregated delay distributions for each combination of `route_desc` (transport type) and `hour` (time rounded to hour). \n",
+    " - `recovery_df3` : contains aggregated delay distributions for `route_desc` (transport type) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>route_int</th>\n",
+       "      <th>trip_int</th>\n",
+       "      <th>stop_int</th>\n",
+       "      <th>stop_sequence</th>\n",
+       "      <th>arrival_time</th>\n",
+       "      <th>departure_time</th>\n",
+       "      <th>route_id</th>\n",
+       "      <th>trip_id</th>\n",
+       "      <th>stop_id</th>\n",
+       "      <th>route_desc</th>\n",
+       "      <th>stop_id_raw</th>\n",
+       "      <th>sequence_shift_1</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>NaT</td>\n",
+       "      <td>2020-05-21 07:18:00</td>\n",
+       "      <td>30-57-Y-j19-1</td>\n",
+       "      <td>4.TA.30-57-Y-j19-1.1.H</td>\n",
+       "      <td>8502208</td>\n",
+       "      <td>Bus</td>\n",
+       "      <td>8502208</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2020-05-21 07:23:00</td>\n",
+       "      <td>2020-05-21 07:23:00</td>\n",
+       "      <td>30-57-Y-j19-1</td>\n",
+       "      <td>4.TA.30-57-Y-j19-1.1.H</td>\n",
+       "      <td>8502209</td>\n",
+       "      <td>Bus</td>\n",
+       "      <td>8502209</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>3</td>\n",
+       "      <td>2020-05-21 07:29:00</td>\n",
+       "      <td>NaT</td>\n",
+       "      <td>30-57-Y-j19-1</td>\n",
+       "      <td>4.TA.30-57-Y-j19-1.1.H</td>\n",
+       "      <td>8503202</td>\n",
+       "      <td>Bus</td>\n",
+       "      <td>8503202</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   route_int  trip_int  stop_int  stop_sequence        arrival_time  \\\n",
+       "0          0         0         0              1                 NaT   \n",
+       "1          0         0         1              2 2020-05-21 07:23:00   \n",
+       "2          0         0         2              3 2020-05-21 07:29:00   \n",
+       "\n",
+       "       departure_time       route_id                 trip_id  stop_id  \\\n",
+       "0 2020-05-21 07:18:00  30-57-Y-j19-1  4.TA.30-57-Y-j19-1.1.H  8502208   \n",
+       "1 2020-05-21 07:23:00  30-57-Y-j19-1  4.TA.30-57-Y-j19-1.1.H  8502209   \n",
+       "2                 NaT  30-57-Y-j19-1  4.TA.30-57-Y-j19-1.1.H  8503202   \n",
+       "\n",
+       "  route_desc stop_id_raw  sequence_shift_1  \n",
+       "0        Bus     8502208                 2  \n",
+       "1        Bus     8502209                 3  \n",
+       "2        Bus     8503202                 1  "
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "###################### MAKE CUMULATIVE PROBABILITY TABLE #######################\n",
+    "\n",
+    "# Load stop_time table, to use its order as a template for our final table \n",
+    "with open(\"../data/stop_times_df.pkl\", \"rb\") as input_file:\n",
+    "    stoptimes = pickle.load(input_file)\n",
+    "    \n",
+    "stoptimes.head(3)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.0%, 4.07%, 8.14%, 12.21%, 16.28%, 20.35%, 24.42%, 28.49%, 32.55%, 36.62%, 40.69%, 44.76%, 48.83%, 52.9%, 56.97%, 61.04%, 65.11%, 69.18%, 73.25%, 77.32%, 81.39%, 85.46%, 89.53%, 93.6%, 97.66%, "
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>key</th>\n",
+       "      <th>key_int</th>\n",
+       "      <th>trip_id</th>\n",
+       "      <th>stop_id</th>\n",
+       "      <th>transport_type</th>\n",
+       "      <th>hour</th>\n",
+       "      <th>distribution</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>4.TA.30-57-Y-j19-1.1.H__8502208</td>\n",
+       "      <td>0</td>\n",
+       "      <td>4.TA.30-57-Y-j19-1.1.H</td>\n",
+       "      <td>8502208</td>\n",
+       "      <td>Bus</td>\n",
+       "      <td>7</td>\n",
+       "      <td>[0, 5, 7, 9, 9, 10, 10, 10, 11, 11, 11, 11, 11...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>4.TA.30-57-Y-j19-1.1.H__8502209</td>\n",
+       "      <td>1</td>\n",
+       "      <td>4.TA.30-57-Y-j19-1.1.H</td>\n",
+       "      <td>8502209</td>\n",
+       "      <td>Bus</td>\n",
+       "      <td>7</td>\n",
+       "      <td>[0, 5, 7, 9, 9, 10, 10, 10, 11, 11, 11, 11, 11...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>4.TA.30-57-Y-j19-1.1.H__8503202</td>\n",
+       "      <td>2</td>\n",
+       "      <td>4.TA.30-57-Y-j19-1.1.H</td>\n",
+       "      <td>8503202</td>\n",
+       "      <td>Bus</td>\n",
+       "      <td>7</td>\n",
+       "      <td>[0, 5, 7, 9, 9, 10, 10, 10, 11, 11, 11, 11, 11...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>5.TA.30-57-Y-j19-1.1.H__8502208</td>\n",
+       "      <td>3</td>\n",
+       "      <td>5.TA.30-57-Y-j19-1.1.H</td>\n",
+       "      <td>8502208</td>\n",
+       "      <td>Bus</td>\n",
+       "      <td>7</td>\n",
+       "      <td>[0, 5, 7, 9, 9, 10, 10, 10, 11, 11, 11, 11, 11...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>5.TA.30-57-Y-j19-1.1.H__8502209</td>\n",
+       "      <td>4</td>\n",
+       "      <td>5.TA.30-57-Y-j19-1.1.H</td>\n",
+       "      <td>8502209</td>\n",
+       "      <td>Bus</td>\n",
+       "      <td>7</td>\n",
+       "      <td>[0, 5, 7, 9, 9, 10, 10, 10, 11, 11, 11, 11, 11...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                               key key_int                 trip_id  stop_id  \\\n",
+       "0  4.TA.30-57-Y-j19-1.1.H__8502208       0  4.TA.30-57-Y-j19-1.1.H  8502208   \n",
+       "1  4.TA.30-57-Y-j19-1.1.H__8502209       1  4.TA.30-57-Y-j19-1.1.H  8502209   \n",
+       "2  4.TA.30-57-Y-j19-1.1.H__8503202       2  4.TA.30-57-Y-j19-1.1.H  8503202   \n",
+       "3  5.TA.30-57-Y-j19-1.1.H__8502208       3  5.TA.30-57-Y-j19-1.1.H  8502208   \n",
+       "4  5.TA.30-57-Y-j19-1.1.H__8502209       4  5.TA.30-57-Y-j19-1.1.H  8502209   \n",
+       "\n",
+       "  transport_type hour                                       distribution  \n",
+       "0            Bus    7  [0, 5, 7, 9, 9, 10, 10, 10, 11, 11, 11, 11, 11...  \n",
+       "1            Bus    7  [0, 5, 7, 9, 9, 10, 10, 10, 11, 11, 11, 11, 11...  \n",
+       "2            Bus    7  [0, 5, 7, 9, 9, 10, 10, 10, 11, 11, 11, 11, 11...  \n",
+       "3            Bus    7  [0, 5, 7, 9, 9, 10, 10, 10, 11, 11, 11, 11, 11...  \n",
+       "4            Bus    7  [0, 5, 7, 9, 9, 10, 10, 10, 11, 11, 11, 11, 11...  "
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "summary_df = pd.DataFrame(columns = ['key', 'key_int', 'trip_id', 'stop_id', 'transport_type', 'hour', 'distribution'])\n",
+    "n_fail = 0\n",
+    "size_stop_times = stoptimes.shape[0]\n",
+    "n_real = 0\n",
+    "n_all  = 0\n",
+    "n_recov1 = 0\n",
+    "n_recov2 = 0\n",
+    "n_recov3 = 0\n",
+    "\n",
+    "for index, row in stoptimes.iterrows():\n",
+    "    \n",
+    "    trip_id = row[7]\n",
+    "    stop_id = str(row[8])[:7]\n",
+    "    transport_type = row[9]\n",
+    "    key = trip_id + '__' + stop_id\n",
+    "\n",
+    "    # Compute rounded hour using arrival if possible - recover with departure\n",
+    "    hour = pd.to_datetime(stoptimes.loc[index]['arrival_time']).hour\n",
+    "    if math.isnan(hour): # if arrival is NaT, use departure time\n",
+    "        hour = pd.to_datetime(stoptimes.loc[index]['departure_time']).hour\n",
+    "    \n",
+    "    distrib = np.zeros(31)\n",
+    "    keep_trying = True\n",
+    "    \n",
+    "    # 1) try d_real to get distribution from measured delays\n",
+    "    if key in d_real:\n",
+    "        distrib = d_real[key]\n",
+    "        sum_distrib = np.sum(distrib)\n",
+    "        if sum_distrib > 100 :\n",
+    "            summary_df.loc[index, 'distribution'] = distrib\n",
+    "            keep_trying = False \n",
+    "            n_real += 1\n",
+    "    \n",
+    "    # 2) try d_all to get distribution from measured + estimated delays\n",
+    "    if keep_trying and key in d_all:\n",
+    "        distrib = d_all[key]\n",
+    "        sum_distrib = np.sum(distrib)\n",
+    "        if sum_distrib > 100 :\n",
+    "            summary_df.loc[index, 'distribution'] = distrib\n",
+    "            keep_trying = False\n",
+    "            n_all += 1\n",
+    "\n",
+    "    # 3) try first recovery table using stop_id, transport_type and hour\n",
+    "    if keep_trying and (stop_id, hour, transport_type) in recovery_df.index:\n",
+    "        distrib = np.array(recovery_df.loc[(stop_id, hour, transport_type)])\n",
+    "        sum_distrib = np.sum(distrib)\n",
+    "        if sum_distrib > 100 :\n",
+    "            summary_df.loc[index, 'distribution'] = distrib\n",
+    "            keep_trying = False \n",
+    "            n_recov1 += 1\n",
+    "            \n",
+    "    # 4) use second recovery table using transport_type and hour \n",
+    "    if keep_trying and (hour, transport_type) in recovery_df2.index:\n",
+    "        distrib = np.array(recovery_df2.loc[(hour, transport_type)])\n",
+    "        sum_distrib = np.sum(distrib)\n",
+    "        if sum_distrib > 100 :\n",
+    "            summary_df.loc[index, 'distribution'] = distrib\n",
+    "            keep_trying = False \n",
+    "            n_recov2 += 1\n",
+    "    \n",
+    "    # 5) use third recovery table using transport_type only \n",
+    "    if keep_trying and (transport_type) in recovery_df3.index:\n",
+    "        distrib = np.array(recovery_df3.loc[(transport_type)])\n",
+    "        sum_distrib = np.sum(distrib)\n",
+    "        summary_df.loc[index, 'distribution'] = distrib\n",
+    "        keep_trying = False \n",
+    "        n_recov3 += 1\n",
+    "            \n",
+    "    # Record results in summary\n",
+    "    summary_df.loc[index, 'key'] = key\n",
+    "    summary_df.loc[index, 'key_int'] = index\n",
+    "    summary_df.loc[index, 'trip_id'] = trip_id\n",
+    "    summary_df.loc[index, 'stop_id'] = stop_id\n",
+    "    summary_df.loc[index, 'transport_type'] = transport_type\n",
+    "    summary_df.loc[index, 'hour'] = hour\n",
+    "\n",
+    "    # save number of failure for recovery\n",
+    "    if keep_trying:\n",
+    "        print('fail{}'.format(index), end = ', ')\n",
+    "        n_fail += 1 \n",
+    "        \n",
+    "    # print progression \n",
+    "    if (index % 10000) == 0 :\n",
+    "        print('{}%'.format(round(100*index/size_stop_times,2)), end = ', ')\n",
+    "    \n",
+    "summary_df.head()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load stop_time table, to use its order as a template for our final table \n",
+    "with gzip.open(\"../data/join_distribution_all.pkl.gz\", \"wb\") as out_file:\n",
+    "    pickle.dump(summary_df, out_file)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([ 0,  5,  7,  9,  9, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, 11,\n",
+       "       11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12])"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "summary_df['distribution'][0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "list_all_rows = []\n",
+    "for index, row in summary_df.iterrows():\n",
+    "    distrib = np.array(row['distribution'])\n",
+    "    \n",
+    "    # get total number of elements \n",
+    "    N = np.sum(distrib)\n",
+    "        \n",
+    "    # make cumulative distribution probabilities\n",
+    "    cdf_distrib = np.empty((len(distrib)), dtype=float)\n",
+    "    save_x = 0\n",
+    "    for x in range(len(distrib)):\n",
+    "        cdf_distrib[x] = float(distrib[x])/float(N) + float(save_x)/float(N)\n",
+    "        save_x += distrib[x]\n",
+    "        \n",
+    "    list_all_rows.append(cdf_distrib)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([[0.        , 0.01501502, 0.03603604, 0.06306306, 0.09009009,\n",
+       "        0.12012012, 0.15015015, 0.18018018, 0.21321321, 0.24624625,\n",
+       "        0.27927928, 0.31231231, 0.34534535, 0.37837838, 0.41141141,\n",
+       "        0.44444444, 0.47747748, 0.51051051, 0.54354354, 0.57657658,\n",
+       "        0.60960961, 0.64264264, 0.67567568, 0.71171171, 0.74774775,\n",
+       "        0.78378378, 0.81981982, 0.85585586, 0.89189189, 0.92792793,\n",
+       "        0.96396396, 1.        ],\n",
+       "       [0.        , 0.01501502, 0.03603604, 0.06306306, 0.09009009,\n",
+       "        0.12012012, 0.15015015, 0.18018018, 0.21321321, 0.24624625,\n",
+       "        0.27927928, 0.31231231, 0.34534535, 0.37837838, 0.41141141,\n",
+       "        0.44444444, 0.47747748, 0.51051051, 0.54354354, 0.57657658,\n",
+       "        0.60960961, 0.64264264, 0.67567568, 0.71171171, 0.74774775,\n",
+       "        0.78378378, 0.81981982, 0.85585586, 0.89189189, 0.92792793,\n",
+       "        0.96396396, 1.        ],\n",
+       "       [0.        , 0.01501502, 0.03603604, 0.06306306, 0.09009009,\n",
+       "        0.12012012, 0.15015015, 0.18018018, 0.21321321, 0.24624625,\n",
+       "        0.27927928, 0.31231231, 0.34534535, 0.37837838, 0.41141141,\n",
+       "        0.44444444, 0.47747748, 0.51051051, 0.54354354, 0.57657658,\n",
+       "        0.60960961, 0.64264264, 0.67567568, 0.71171171, 0.74774775,\n",
+       "        0.78378378, 0.81981982, 0.85585586, 0.89189189, 0.92792793,\n",
+       "        0.96396396, 1.        ],\n",
+       "       [0.        , 0.01501502, 0.03603604, 0.06306306, 0.09009009,\n",
+       "        0.12012012, 0.15015015, 0.18018018, 0.21321321, 0.24624625,\n",
+       "        0.27927928, 0.31231231, 0.34534535, 0.37837838, 0.41141141,\n",
+       "        0.44444444, 0.47747748, 0.51051051, 0.54354354, 0.57657658,\n",
+       "        0.60960961, 0.64264264, 0.67567568, 0.71171171, 0.74774775,\n",
+       "        0.78378378, 0.81981982, 0.85585586, 0.89189189, 0.92792793,\n",
+       "        0.96396396, 1.        ],\n",
+       "       [0.        , 0.01501502, 0.03603604, 0.06306306, 0.09009009,\n",
+       "        0.12012012, 0.15015015, 0.18018018, 0.21321321, 0.24624625,\n",
+       "        0.27927928, 0.31231231, 0.34534535, 0.37837838, 0.41141141,\n",
+       "        0.44444444, 0.47747748, 0.51051051, 0.54354354, 0.57657658,\n",
+       "        0.60960961, 0.64264264, 0.67567568, 0.71171171, 0.74774775,\n",
+       "        0.78378378, 0.81981982, 0.85585586, 0.89189189, 0.92792793,\n",
+       "        0.96396396, 1.        ]])"
+      ]
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df = pd.DataFrame(list_all_rows)\n",
+    "final_df.index = summary_df.index\n",
+    "final_np = final_df.to_numpy()\n",
+    "final_np[0:5,:]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 40,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sum(np.array(final_df.index == stoptimes.index)) == stoptimes.shape[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 104,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# write recovery table \n",
+    "with gzip.open(\"../data/distrib_recov_tab_stopID_hour.pkl.gz\", \"wb\") as output_file:\n",
+    "    pickle.dump(recovery_tab, output_file)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "### Poisson cumulative distribution\n",
     "\n",
     "The Poisson distribution is popular for modeling the number of times an event occurs in an interval of time or space. We modeled a poisson distribution for delays assuming parameter $k$ is the time in minutes (as it was done [here](https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0126137), formulas $(4),(5),(6)$).\n",
     "\n",
     "A discrete random variable X is said to have a Poisson distribution with parameter λ > 0, if, for k = 0, 1, 2, ..., the probability mass function of X is given by:\n",
     "\n",
     "$${\\displaystyle \\!f(k;\\lambda )=\\Pr(X=k)={\\frac {\\lambda ^{k}e^{-\\lambda }}{k!}},}$$\n",
     "where\n",
     "\n",
     "e is Euler's number (e = 2.71828...)\n",
     "k! is the factorial of k.\n",
     "The positive real number λ is equal to the expected value of X __and__ to its variance.\n",
     "\n",
     "$${\\displaystyle \\lambda =\\operatorname {E} (X)=\\operatorname {Var} (X)}$$\n",
     "\n",
     "We can approximate E[𝑋]∼$\\mu_i$ for our data $X_i$, if we assume the sample $X_i$ of size N follow the distribution of $X$ meaning $X_i$∼$X$.\n",
     "\n",
     "Poisson-related __assumptions__ :\n",
     "- $k$ is the __delay time in minutes__ and can take values 0, 1, 2, ... (strictly positive and discrete)\n",
     "- We assume our sampling $X_i$ of $X$ is good enough to approximate E[X] ~ $\\mu_i$\n",
     "- The occurrence of one event does not affect probability of others. That is, events occur independently.\n",
     "    - __We assume being late one day is not affecting the delay of the day after__ \n",
     "- The average rate at which events occur is independent of any occurrences. For simplicity, this is usually assumed to be constant, but may in practice vary with time.\n",
     "    - __we assumes delays occurs with a constant rate over time__\n",
     "- Two events cannot occur at exactly the same instant\n",
     "\n",
     "We made a function `poisson_proba` that takes a `trip_id`, a `stop_id`, an `arrival time` and a `departure time` and a dictionnary {key : distribution} to compute a __probability to be at least 2 minutes before departure of next trip__. \n",
     "\n",
     "We make a few __assumptions__ on our side :\n",
     "- We assume that if we have less than 2 minutes for the transfer, we miss it.\n",
     "- We assume the next train is on time.\n",
     "- As for poisson distribution $k$ is strictly positive, we assume trains ahead of schedule were on time ($k=0$)\n",
     "\n",
     "\n",
     "_Question we should address :_\n",
     "- _Is the poisson a reasonable approximation of the binomial distribution in our case ?_"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "Let's first test the poisson distribution and compare it with our distribution to see how well it fits the data. We will compute $Pr(X = k)$ for each values of k and look at the shape of the poisson distribution compared to the shape of our scaled data. Then, we will compare $\\sum_{k=0}^T Pr(X = k)$ with the cumulative distribution function which directly gives $Pr(k \\leq X)$"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 22,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
       "An error was encountered:\n",
       "Invalid status code '404' from http://iccluster044.iccluster.epfl.ch:8998/sessions/6821 with error payload: \"Session '6821' not found.\"\n"
      ]
     }
    ],
    "source": [
     "################################# POISSON FIT TEST #########################################\n",
     "\n",
     "# to do .. \n"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "Here are all the functions needed to calculate probability of success for a given transfer. We need the `trip_id`, `stop_id`, `departure_time`, `arrival_time` and dictionnary `d` (pickled load at the beginning of the cell) to be able to compute a probability of success with following function : \n",
     "\n",
     "`poisson_proba(trip_id, stop_id, arrival_time, departure_time, d)`"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 40,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "lambda (expectation given distribution):  1.0194769059543685 \n",
       "\n",
       "Probability of success for transfer time = 13.0 minutes :  0.999999999994185\n"
      ]
     }
    ],
    "source": [
     "%local\n",
     "################################# POISSON FUNCTIONS ########################################\n",
     "\n",
     "import pickle \n",
     "import gzip\n",
     "import time\n",
     "import math \n",
     "import datetime\n",
     "import time\n",
     "from scipy.stats import poisson\n",
     "\n",
     "# Load dictionnary\n",
     "with gzip.open(\"../data/distributions.pickle\", \"rb\") as input_file:\n",
     "    d = pickle.load(input_file)\n",
     "\n",
     "# Load dictionnary\n",
     "with open(\"../data/stop_times_array.pkl\", \"rb\") as input_file:\n",
     "    times = pickle.load(input_file)\n",
     "\n",
     "# we take two exemple time in format numpy.datetime64\n",
     "arr_time = times[4][1]\n",
     "dep_time = times[0][1]\n",
     "\n",
     "# Load distribution in dictinonary given a key\n",
     "def get_distrib(key, dico):\n",
     "    if key in dico:\n",
     "        return dico[key]\n",
     "    else:\n",
     "        raise ValueError(\"KEY ERROR: {} not found un distribution dictionnary\".format(key))\n",
     "    \n",
     "# Evaluate lambda parameter assuming it is equal to average \n",
     "def evaluate_lamda(distrib):\n",
     "    # First calculate total number of measures N\n",
     "    N = 0 # by starting at -1 we ignore trains ahead of schedule\n",
     "    for x in distrib:\n",
     "        N += x\n",
     "\n",
     "    lambda_p = 0 # expectation - we want to calculate it\n",
     "    t        = -1 # time = index - 1\n",
     "\n",
     "    for x in distrib:\n",
     "        if t>0:\n",
     "            lambda_p += t*x\n",
     "        t += 1\n",
     "\n",
     "    # calculate lambda - the expectation of x\n",
     "    if N > 0:\n",
     "        lambda_p /= N \n",
     "        print('lambda (expectation given distribution): ',lambda_p, '\\n')\n",
     "        return lambda_p\n",
     "    else : \n",
     "        raise ValueError(\"ERROR : {} distribution has 0 counts\".format(key))\n",
     "        #print('Returning 1 to avoid later problem... \\n')\n",
     "        return 1\n",
     "\n",
     "# process time given as string in format 'hh:mm' - not needed\n",
     "def process_time_str(str_time):\n",
     "    x = time.strptime(str_time,'%H:%M')\n",
     "    return datetime.timedelta(hours=x.tm_hour,minutes=x.tm_min,seconds=x.tm_sec).total_seconds()\n",
     "\n",
     "# Calculate transfer time given two times in string format 'hh:mm'\n",
     "def get_transfer_time(arr_time, dep_time, delta=2.0):\n",
     "    diff_time_min = (arr_time - dep_time).astype('timedelta64[m]') / np.timedelta64(1, 'm')\n",
     "    return diff_time_min - delta\n",
     "\n",
     "# Calculate poisson probability of success for a given transfert \n",
     "# for a given trip_id, stop_id, arrival/departure times and dict\n",
     "def poisson_proba(trip_id, stop_id, arr_time, dep_time, dico):\n",
     "    # Generate key from trip_id / stop_id \n",
     "    key = str(trip_id) + '__' + str(stop_id[0:7]) # 7 first char to be sbb-compatible\n",
     "\n",
     "    # Get distribution from dictionnary\n",
     "    distrib = get_distrib(key, dico)\n",
     "    \n",
     "    # Calculate transfer time at disposal \n",
     "    T = get_transfer_time(arr_time, dep_time)\n",
     "    \n",
     "    # Get lambda value to calculate proba\n",
     "    lambda_p = evaluate_lamda(distrib)\n",
     "\n",
     "    # Get proba\n",
     "    if T > 2:\n",
     "        poisson_p = poisson.cdf(T, lambda_p)\n",
     "    else : \n",
     "        poisson_p = 0.0 # if we have less than 2 minutes, we miss it\n",
     "        \n",
     "    print('Probability of success for transfer time = {} minutes : '.format(T),poisson_p)\n",
     "    return poisson_p\n",
     "\n",
     "# Mock exemple of probability calculations with given inputs\n",
     "trip_id = '1286.TA.26-32-j19-1.12.H'\n",
     "stop_id = '8591184'\n",
     "\n",
     "# we take two exemple time from stop_times_array in format numpy.datetime64\n",
     "arr_time = times[3][1]\n",
     "dep_time = times[0][1]\n",
     "\n",
     "Pr = poisson_proba(trip_id, stop_id, arr_time, dep_time, d)\n"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
    "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
     "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.7.6"
   }
  },
  "nbformat": 4,
  "nbformat_minor": 4
 }
diff --git a/notebooks/probabilities.ipynb b/notebooks/probabilities.ipynb
index 915b88a..faf0c2e 100644
--- a/notebooks/probabilities.ipynb
+++ b/notebooks/probabilities.ipynb
@@ -1,2251 +1,2476 @@
 {
  "cells": [
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "## Make distribution tables to calculate probabilities of transfer\n",
     "\n",
     "<div class='alert alert-info'><b>Any application without a proper name would be promptly killed.</b></div>"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "Current session configs: <tt>{'conf': {'spark.app.name': 'lgptguys_final'}, 'kind': 'pyspark'}</tt><br>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table>\n",
+       "<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>7272</td><td>application_1589299642358_1768</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1768/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster068.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1768_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7292</td><td>application_1589299642358_1788</td><td>pyspark</td><td>busy</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1788/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster072.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1788_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7326</td><td>application_1589299642358_1822</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1822/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster067.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1822_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7369</td><td>application_1589299642358_1865</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1865/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster072.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1865_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7388</td><td>application_1589299642358_1884</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1884/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster072.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1884_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7393</td><td>application_1589299642358_1889</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1889/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster067.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1889_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7398</td><td>application_1589299642358_1894</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1894/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster068.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1894_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7407</td><td>application_1589299642358_1903</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1903/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster065.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1903_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7412</td><td>application_1589299642358_1908</td><td>pyspark</td><td>busy</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1908/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster071.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1908_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7415</td><td>application_1589299642358_1911</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1911/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster069.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1911_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7418</td><td>application_1589299642358_1914</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1914/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster070.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1914_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7420</td><td>application_1589299642358_1916</td><td>pyspark</td><td>busy</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1916/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster071.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1916_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7421</td><td>application_1589299642358_1917</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1917/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster070.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1917_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7422</td><td>application_1589299642358_1918</td><td>pyspark</td><td>busy</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1918/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster068.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1918_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7423</td><td>application_1589299642358_1919</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1919/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster067.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1919_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7424</td><td>application_1589299642358_1920</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1920/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster071.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1920_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7426</td><td>application_1589299642358_1922</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1922/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster069.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1922_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7427</td><td>application_1589299642358_1923</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1923/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster069.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1923_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7428</td><td>application_1589299642358_1924</td><td>pyspark</td><td>busy</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1924/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster067.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1924_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7429</td><td>application_1589299642358_1925</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1925/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster066.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1925_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7431</td><td>application_1589299642358_1927</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1927/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster067.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1927_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7433</td><td>application_1589299642358_1929</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1929/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster070.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1929_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7434</td><td>application_1589299642358_1930</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1930/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster067.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1930_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7435</td><td>application_1589299642358_1931</td><td>pyspark</td><td>busy</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1931/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster066.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1931_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7437</td><td>application_1589299642358_1933</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1933/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster066.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1933_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7438</td><td>application_1589299642358_1934</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1934/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster068.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1934_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7440</td><td>application_1589299642358_1936</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1936/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster072.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1936_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7441</td><td>application_1589299642358_1937</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1937/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster065.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1937_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7443</td><td>application_1589299642358_1939</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1939/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster065.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1939_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7444</td><td>application_1589299642358_1940</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1940/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster065.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1940_01_000001/ebouille\">Link</a></td><td></td></tr><tr><td>7445</td><td>application_1589299642358_1941</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1941/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster065.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1941_01_000001/ebouille\">Link</a></td><td></td></tr></table>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "%%configure\n",
     "{\"conf\": {\n",
     "    \"spark.app.name\": \"lgptguys_final\"\n",
     "}}"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "### Start Spark"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Initialization\n",
-    "%%spark"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 57,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Starting Spark application\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table>\n",
+       "<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>7446</td><td>application_1589299642358_1942</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://iccluster044.iccluster.epfl.ch:8088/proxy/application_1589299642358_1942/\">Link</a></td><td><a target=\"_blank\" href=\"http://iccluster065.iccluster.epfl.ch:8042/node/containerlogs/container_e06_1589299642358_1942_01_000001/ebouille\">Link</a></td><td>✔</td></tr></table>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Successfully passed 'username' as 'username' to Spark kernel"
+      "SparkSession available as 'spark'.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "An error was encountered:\n",
+      "unknown magic command '%spark'\n",
+      "UnknownMagic: unknown magic command '%spark'\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Initialization\n",
+    "%%spark"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "An error was encountered:\n",
+      "Variable named username not found.\n"
      ]
     }
    ],
    "source": [
     "%%send_to_spark -i username -t str -n username"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "### Import useful libraries "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 58,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     }
    ],
    "source": [
     "from geopy.distance import great_circle\n",
     "from pyspark.sql.functions import *\n",
     "from pyspark.sql.types import StructType, StructField, StringType, IntegerType, LongType\n"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "### Read TimeTable data for routes / trips "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 59,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "+-----------+--------+---------------------------+-------+\n",
-      "|stop_id_raw|stop_int|stop_name                  |stop_id|\n",
-      "+-----------+--------+---------------------------+-------+\n",
-      "|8500926    |0       |Oetwil a.d.L., Schweizäcker|8500926|\n",
-      "|8502186    |1       |Dietikon Stoffelbach       |8502186|\n",
-      "|8502186:0:1|2       |Dietikon Stoffelbach       |8502186|\n",
-      "|8502186:0:2|3       |Dietikon Stoffelbach       |8502186|\n",
-      "|8502186P   |4       |Dietikon Stoffelbach       |8502186|\n",
-      "+-----------+--------+---------------------------+-------+\n",
+      "+-----------+---------------------------+-------+\n",
+      "|stop_id_raw|stop_name                  |stop_id|\n",
+      "+-----------+---------------------------+-------+\n",
+      "|8500926    |Oetwil a.d.L., Schweizäcker|8500926|\n",
+      "|8502186    |Dietikon Stoffelbach       |8502186|\n",
+      "|8502186:0:1|Dietikon Stoffelbach       |8502186|\n",
+      "|8502186:0:2|Dietikon Stoffelbach       |8502186|\n",
+      "|8502186P   |Dietikon Stoffelbach       |8502186|\n",
+      "+-----------+---------------------------+-------+\n",
       "only showing top 5 rows"
      ]
     }
    ],
    "source": [
     "stops_15km        = spark.read.csv('data/lgpt_guys/stops_15km.csv', header = True)\n",
     "\n",
     "# We use only first 7 characters of stop_id to remove special cases\n",
-    "stops_15km = stops_15km.select(col('stop_id').alias('stop_id_raw'), 'stop_int', 'stop_name')\\\n",
+    "stops_15km = stops_15km.select(col('stop_id').alias('stop_id_raw'), 'stop_name')\\\n",
     "                       .withColumn('stop_id',col('stop_id_raw').substr(1, 7))\n",
     "stops_15km.show(5, False)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "### Read the [SBB actual data](https://opentransportdata.swiss/en/dataset/istdaten) in ORC format"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     }
    ],
    "source": [
     "sbb = spark.read.orc('/data/sbb/orc/istdaten')"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "root\n",
       " |-- betriebstag: string (nullable = true)\n",
       " |-- fahrt_bezeichner: string (nullable = true)\n",
       " |-- betreiber_id: string (nullable = true)\n",
       " |-- betreiber_abk: string (nullable = true)\n",
       " |-- betreiber_name: string (nullable = true)\n",
       " |-- produkt_id: string (nullable = true)\n",
       " |-- linien_id: string (nullable = true)\n",
       " |-- linien_text: string (nullable = true)\n",
       " |-- umlauf_id: string (nullable = true)\n",
       " |-- verkehrsmittel_text: string (nullable = true)\n",
       " |-- zusatzfahrt_tf: string (nullable = true)\n",
       " |-- faellt_aus_tf: string (nullable = true)\n",
       " |-- bpuic: string (nullable = true)\n",
       " |-- haltestellen_name: string (nullable = true)\n",
       " |-- ankunftszeit: string (nullable = true)\n",
       " |-- an_prognose: string (nullable = true)\n",
       " |-- an_prognose_status: string (nullable = true)\n",
       " |-- abfahrtszeit: string (nullable = true)\n",
       " |-- ab_prognose: string (nullable = true)\n",
       " |-- ab_prognose_status: string (nullable = true)\n",
       " |-- durchfahrt_tf: string (nullable = true)"
      ]
     }
    ],
    "source": [
     "sbb.printSchema()"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "### Subset SBB data\n",
     "\n",
     "We take only stop_id in 15 km range from Zurich HB using `stop_id` field from _stops_15km_. We did not use only `geschaetz` prognose time as there was too few overlap between _timetable_ and _sbb_ datasets with only `geschaetz` arrival times. _To do next : Use only geschaetz when available_"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "218697932\n",
+      "10848628\n",
       "+----------------+-----------------+----------------+----------------+-------------------+-------------------+-------+\n",
       "|fahrt_bezeichner|haltestellen_name|ankunftszeit    |abfahrtszeit    |an_prognose        |ab_prognose        |stop_id|\n",
       "+----------------+-----------------+----------------+----------------+-------------------+-------------------+-------+\n",
       "|85:11:10:002    |Zürich HB        |03.09.2018 21:51|                |03.09.2018 21:53:40|                   |8503000|\n",
       "|85:11:11:001    |Zürich HB        |                |03.09.2018 06:09|                   |03.09.2018 06:10:22|8503000|\n",
       "|85:11:12:001    |Zürich HB        |03.09.2018 10:51|                |03.09.2018 10:51:28|                   |8503000|\n",
       "|85:11:1251:003  |Zürich HB        |03.09.2018 07:00|                |03.09.2018 07:00:01|                   |8503000|\n",
       "|85:11:1252:001  |Zürich HB        |03.09.2018 21:23|03.09.2018 21:36|03.09.2018 21:24:55|03.09.2018 21:36:57|8503000|\n",
       "+----------------+-----------------+----------------+----------------+-------------------+-------------------+-------+\n",
       "only showing top 5 rows"
      ]
     }
    ],
    "source": [
     "# Used to subset sbb table based on stop_id \n",
     "l1_id = stops_15km.select('stop_id').collect()\n",
     "l2_id = [item.stop_id for item in l1_id]\n",
     "\n",
     "# Used to subset sbb table based on stop_names \n",
     "l1_name = stops_15km.select('stop_name').collect()\n",
     "l2_name = [item.stop_name for item in l1_name]\n",
     "\n",
     "# Make the subset dataframe\n",
-    "sbb_filt = sbb.filter( sbb['bpuic'].isin(l2_id) | sbb['bpuic'].isin(l2_name) ) \\\n",
+    "sbb_filt = sbb.filter( ( sbb['bpuic'].isin(l2_id) | sbb['bpuic'].isin(l2_name) ) &\\\n",
+    "                       ((sbb.an_prognose_status == 'REAL') | \\\n",
+    "                        (sbb.an_prognose_status == 'GESCHAETZ') | \\\n",
+    "                        (sbb.ab_prognose_status == 'REAL') | \\\n",
+    "                        (sbb.ab_prognose_status == 'GESCHAETZ') ) ) \\\n",
     "              .select('fahrt_bezeichner','haltestellen_name', \\\n",
     "                      'ankunftszeit', 'abfahrtszeit', \\\n",
     "                      'an_prognose', 'ab_prognose', \\\n",
     "                      col('bpuic').alias('stop_id'))\n",
     "\n",
-    "#                       ((sbb.an_prognose_status == 'GESCHAETZT') | \\\n",
-    "#                       ( sbb.ab_prognose_status == 'GESCHAETZT') ) \n",
-    "#                       sbb(\"betriebstag\").gt(lit(\"2019-01-01\")) &\\\n",
-    "#                       sbb(\"betriebstag\").lt(lit(\"2019-06-30\")) &\\\n",
-    "                       \n",
     "print sbb_filt.count()\n",
     "sbb_filt.show(5,False)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "Write subset table in HDFS for better performance during later usage"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     }
    ],
    "source": [
     "# save\n",
     "username = 'acoudray'\n",
-    "sbb_filt.write.format(\"orc\").save(\"/user/{}/sbb_filt_forDelays_noGaeschetz.orc\".format(username))"
+    "sbb_filt.write.format(\"orc\").save(\"/user/{}/sbb_filt_forDelays_GeschaetzAndReal.orc\".format(username))"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "Summary of tables writen in /user/{}/ :\n",
     "- sbb_filt_forDelays_noGaeschetz.orc : table with all dates, < 15km, no GESCHAETZ, used 7-char trimmed stop_id in timetable data\n",
     "- sbb_filt_forDelays2.orc : table with all dates, < 15km, only GESCHAETZ, used 7-char trimmed stop_id in timetable data\n",
     "- sbb_filt_forDelays.orc : table with all dates, < 15km, only GESCHAETZ\n",
     "- sbb_sub_forDelays.ord  : Old to remove"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "### Measure Distributions of Delay Times per trip and station\n",
     "\n",
     "The goal of this chapter is to pre-compute probabilities for McRaptor implementation, which will be ultimately used to choose the best trip according to its time __and probability of success__. The goal is to create a distribution of arrival delays for each station / trip_id pair. \n",
     "\n",
     "We begin with a simple query of trip_id / station_id and build up to the full table generation made from correspondance tables between sbb and timetable trip_ids (they need to be translated first, which is done in `match_datasets.ipynb`.\n",
     "\n",
     "#### Simple task : returning the distribution for a given station / trip id\n",
     "\n",
     "Let's begin by exploring _sbb_ data and compute a distribution step by step for a given station / trip_id "
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 20,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "2133164"
      ]
     }
    ],
    "source": [
     "# Load sbb data \n",
     "username='acoudray'\n",
     "sbb = spark.read.orc(\"/user/{}/sbb_filt_forDelays.orc\".format(username))\n",
     "sbb.count()"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 21,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "+-------------------+\n",
       "|  haltestellen_name|\n",
       "+-------------------+\n",
       "|Winkel am Zürichsee|\n",
       "|   Zürich Flughafen|\n",
       "|          Kemptthal|\n",
       "|             Urdorf|\n",
       "|    Zürich Wiedikon|\n",
       "+-------------------+\n",
       "only showing top 5 rows"
      ]
     }
    ],
    "source": [
     "sbb.select(\"haltestellen_name\").distinct().show(5)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "Here we show the first few lines of all unique stations. We pick one of them and show its first associated trip id."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 22,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "+----------------+\n",
       "|fahrt_bezeichner|\n",
       "+----------------+\n",
       "|  85:11:1507:002|\n",
       "|  85:11:1509:003|\n",
       "|  85:11:1510:003|\n",
       "|  85:11:1511:003|\n",
       "|  85:11:1512:003|\n",
       "+----------------+\n",
       "only showing top 5 rows"
      ]
     }
    ],
    "source": [
     "stop=\"Zürich Flughafen\"\n",
     "sbb.filter(sbb.haltestellen_name == stop).select(\"fahrt_bezeichner\").show(5)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 24,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "+----------------+--------------+-------------------+-------------+-------------+-------+\n",
       "|         station|       trip_id|       arrival_true|DiffInSeconds|DiffInMinutes|weekday|\n",
       "+----------------+--------------+-------------------+-------------+-------------+-------+\n",
       "|Zürich Flughafen|85:11:1507:002|2018-05-06 06:49:24|           24|          0.0|    Sun|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-05-05 06:49:15|           15|          0.0|    Sat|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-05-04 06:50:38|           98|          2.0|    Fri|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-05-03 06:50:11|           71|          1.0|    Thu|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-05-02 06:49:30|           30|          1.0|    Wed|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-05-01 06:49:38|           38|          1.0|    Tue|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-04-30 06:49:59|           59|          1.0|    Mon|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-04-29 06:49:16|           16|          0.0|    Sun|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-04-28 06:49:37|           37|          1.0|    Sat|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-04-27 06:50:00|           60|          1.0|    Fri|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-04-26 06:49:58|           58|          1.0|    Thu|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-04-25 06:49:44|           44|          1.0|    Wed|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-04-24 06:50:10|           70|          1.0|    Tue|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-04-23 06:49:53|           53|          1.0|    Mon|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-04-22 06:49:33|           33|          1.0|    Sun|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-04-21 06:49:00|            0|          0.0|    Sat|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-04-20 06:49:43|           43|          1.0|    Fri|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-04-19 06:49:00|            0|          0.0|    Thu|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-04-18 06:49:39|           39|          1.0|    Wed|\n",
       "|Zürich Flughafen|85:11:1507:002|2018-04-17 06:49:36|           36|          1.0|    Tue|\n",
       "+----------------+--------------+-------------------+-------------+-------------+-------+\n",
       "only showing top 20 rows"
      ]
     }
    ],
    "source": [
     "trip_id=\"85:11:1507:002\"\n",
     "\n",
     "# First filter - filter selected station/trip_id, with define arrival time and GAESCHETZ status\n",
     "# Select 4 fields of interest, rename \n",
     "# Convert date-like string to timestamp\n",
     "# Compute difference between scheduled and actual arrivals times\n",
     "# reselect to generate weekday\n",
     "sbb_filt = sbb.filter( (sbb.fahrt_bezeichner == trip_id) & (sbb.haltestellen_name == stop) )\\\n",
     "                .select(col(\"haltestellen_name\").alias(\"station\"), \\\n",
     "                        col(\"fahrt_bezeichner\").alias(\"trip_id\"), \\\n",
     "                        col(\"an_prognose\").alias(\"arrival_true\"),\\\n",
     "                        col(\"ankunftszeit\").alias(\"arrival_expected\"))\\\n",
     "                .withColumn('arrival_true',to_timestamp(col('arrival_true'),\\\n",
     "                                                          format='dd.MM.yyyy HH:mm:ss'))\\\n",
     "                .withColumn('arrival_expected',to_timestamp(col('arrival_expected'),\\\n",
     "                                                           format='dd.MM.yyyy HH:mm'))\\\n",
     "                .withColumn('DiffInSeconds',col('arrival_true').cast(LongType()) - col('arrival_expected').cast(LongType()))\\\n",
     "                .withColumn('DiffInMinutes',round(col('DiffInSeconds')/60))\\\n",
     "                .select(\"station\", \"trip_id\", \"arrival_true\", \"DiffInSeconds\", \"DiffInMinutes\",\\\n",
     "                        date_format('arrival_expected', 'E').alias('weekday'))\\\n",
     "                .orderBy(\"arrival_true\", ascending=False)\n",
     "sbb_filt.show()"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "Given a station name and a trip id, we can get all arrival times (prognosed and real), and compute all delays in seconds and minutes. As we see the expected arrival time `ankunftzeit` is always the same as opposed to the actual arrival `an_prognose` with `an_prognose_status` equal to `GESCHAETZT` which varies.\n",
     "\n",
     "We remove Saturdays and Sundays to compute the arrival distribution only based on week days "
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 25,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "+-------------+-----+\n",
       "|DiffInMinutes|count|\n",
       "+-------------+-----+\n",
       "|          0.0|   19|\n",
       "|          1.0|   50|\n",
       "|          2.0|   14|\n",
       "|          3.0|    4|\n",
       "|          4.0|    1|\n",
       "|         11.0|    1|\n",
       "|         21.0|    1|\n",
       "+-------------+-----+"
      ]
     }
    ],
    "source": [
     "sbb_filt.filter( (sbb_filt.weekday != \"Sun\") & (sbb_filt.weekday != \"Sat\") )\\\n",
     "        .groupBy('DiffInMinutes').count()\\\n",
     "        .orderBy(\"DiffInMinutes\").show()"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "For next steps, we will be able to pivot this kind of table for multiple trip ids at once. "
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "#### Load Table and make distribution from list of stations/trip_id\n",
     "\n",
     "Here we compute distribution of delays for a group of stations with all associated trips. The goal is to develop a script able to make a distribution for all stations/trips of interests.\n",
     "\n",
     "To train a bit the concept, let's first use all station with _Zurich_ pattern in their name and compute their delay distribution."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 26,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "+--------------------+\n",
       "|   haltestellen_name|\n",
       "+--------------------+\n",
       "| Winkel am Zürichsee|\n",
       "|    Zürich Flughafen|\n",
       "|     Zürich Wiedikon|\n",
       "|  Zürich Stadelhofen|\n",
       "|Zürich Tiefenbrunnen|\n",
       "+--------------------+\n",
       "only showing top 5 rows"
      ]
     }
    ],
    "source": [
     "expr = \"Z.rich*\" # regular expression to be used to get all Zurich* stations\n",
     "sbb.filter(sbb[\"haltestellen_name\"].rlike(expr))\\\n",
     "    .select(\"haltestellen_name\")\\\n",
     "    .distinct().show(5)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 28,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "+--------------------+---------------+-------------------+-------------+-------------+-------+\n",
       "|             station|        trip_id|       arrival_true|DiffInSeconds|DiffInMinutes|weekday|\n",
       "+--------------------+---------------+-------------------+-------------+-------------+-------+\n",
       "|           Zürich HB|  85:11:543:001|2018-05-07 01:12:14|           14|          0.0|    Mon|\n",
       "|           Zürich HB|85:11:30797:011|2018-05-07 00:58:45|          225|          4.0|    Mon|\n",
       "|           Zürich HB|85:11:19694:001|2018-05-07 00:56:12|          -48|         -1.0|    Mon|\n",
       "|           Zürich HB| 85:11:2294:003|2018-05-07 00:54:06|          -54|         -1.0|    Mon|\n",
       "|   Zürich Hardbrücke|85:11:30797:011|2018-05-07 00:54:03|          123|          2.0|    Mon|\n",
       "|  Zürich Stadelhofen|85:11:19694:001|2018-05-07 00:53:40|           40|          1.0|    Mon|\n",
       "|   Zürich Hardbrücke|85:11:30794:002|2018-05-07 00:50:26|           86|          1.0|    Mon|\n",
       "|     Zürich Oerlikon|85:11:30797:011|2018-05-07 00:49:46|          106|          2.0|    Mon|\n",
       "|Zürich Tiefenbrunnen|85:11:19694:001|2018-05-07 00:49:38|          -22|          0.0|    Mon|\n",
       "|           Zürich HB|85:11:30794:002|2018-05-07 00:47:16|          136|          2.0|    Mon|\n",
       "|           Zürich HB|85:11:30692:007|2018-05-07 00:45:02|          242|          4.0|    Mon|\n",
       "|           Zürich HB|85:11:20495:001|2018-05-07 00:45:01|          -59|         -1.0|    Mon|\n",
       "|   Zürich Altstetten|85:11:18594:001|2018-05-07 00:44:53|           53|          1.0|    Mon|\n",
       "|  Zürich Stadelhofen|85:11:30794:002|2018-05-07 00:44:27|          147|          2.0|    Mon|\n",
       "|  Zürich Stadelhofen|85:11:30692:007|2018-05-07 00:42:37|          277|          5.0|    Mon|\n",
       "|           Zürich HB| 85:11:4793:001|2018-05-07 00:42:10|           10|          0.0|    Mon|\n",
       "|    Zürich Flughafen| 85:11:2294:003|2018-05-07 00:42:07|         -113|         -2.0|    Mon|\n",
       "|   Zürich Hardbrücke|85:11:18594:001|2018-05-07 00:41:27|           27|          0.0|    Mon|\n",
       "|           Zürich HB|85:11:18795:001|2018-05-07 00:41:05|           65|          1.0|    Mon|\n",
       "|    Zürich Wipkingen|85:11:20495:001|2018-05-07 00:40:52|           52|          1.0|    Mon|\n",
       "+--------------------+---------------+-------------------+-------------+-------------+-------+\n",
       "only showing top 20 rows"
      ]
     }
    ],
    "source": [
     "expr = \"Z.rich*\"\n",
     "\n",
     "# First filter - Take Zurich-like stations , with define arrival time and GAESCHETZ status\n",
     "# Select 4 fields of interest, rename \n",
     "# Convert date-like string to timestamp\n",
     "# Compute difference between scheduled and actual arrivals times\n",
     "# reselect to generate weekday\n",
     "sbb_filt = sbb.filter((sbb[\"haltestellen_name\"].rlike(expr)) )\\\n",
     "                .select(col(\"haltestellen_name\").alias(\"station\"), \\\n",
     "                        col(\"fahrt_bezeichner\").alias(\"trip_id\"), \\\n",
     "                        col(\"an_prognose\").alias(\"arrival_true\"),\\\n",
     "                        col(\"ankunftszeit\").alias(\"arrival_expected\"))\\\n",
     "                .withColumn('arrival_true',to_timestamp(col('arrival_true'),\\\n",
     "                                                          format='dd.MM.yyyy HH:mm:ss'))\\\n",
     "                .withColumn('arrival_expected',to_timestamp(col('arrival_expected'),\\\n",
     "                                                           format='dd.MM.yyyy HH:mm'))\\\n",
     "                .withColumn('DiffInSeconds',col('arrival_true').cast(LongType()) - col('arrival_expected').cast(LongType()))\\\n",
     "                .withColumn('DiffInMinutes',round(col('DiffInSeconds')/60))\\\n",
     "                .select(\"station\", \"trip_id\", \"arrival_true\", \"DiffInSeconds\", \"DiffInMinutes\",\\\n",
     "                        date_format('arrival_expected', 'E').alias('weekday'))\\\n",
     "                .orderBy(\"arrival_true\", ascending=False)\n",
     "\n",
     "# Remove Saturday and Sunday weekdays from table - show\n",
     "sbb_filt = sbb_filt.filter( (sbb_filt.weekday != \"Sun\") & (sbb_filt.weekday != \"Sat\") )\n",
     "sbb_filt.show()"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "To make distribution, we use groupBy followed by a pivot using delay time in minutes. We fill null values with 0. No lower/upper bounds for now. Negative column keys means arrival ahead of schedule."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 29,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "+--------------------+---------------+----+-----+-----+-----+-----+-----+-----+----+----+----+----+----+----+----+----+----+---+---+---+---+---+---+---+---+---+---+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+\n",
       "|             station|        trip_id|null|-28.0|-15.0|-13.0|-12.0|-11.0|-10.0|-9.0|-8.0|-7.0|-6.0|-5.0|-4.0|-3.0|-2.0|-1.0|0.0|1.0|2.0|3.0|4.0|5.0|6.0|7.0|8.0|9.0|10.0|11.0|12.0|13.0|14.0|15.0|16.0|17.0|18.0|19.0|20.0|21.0|22.0|23.0|24.0|25.0|26.0|27.0|28.0|29.0|30.0|31.0|32.0|33.0|34.0|35.0|36.0|37.0|38.0|39.0|40.0|41.0|42.0|43.0|44.0|45.0|46.0|47.0|48.0|49.0|50.0|51.0|52.0|53.0|54.0|55.0|56.0|57.0|59.0|60.0|61.0|62.0|63.0|64.0|65.0|66.0|67.0|68.0|69.0|70.0|71.0|72.0|73.0|76.0|77.0|78.0|79.0|80.0|82.0|85.0|86.0|90.0|96.0|99.0|102.0|111.0|120.0|122.0|127.0|132.0|149.0|150.0|152.0|180.0|210.0|\n",
       "+--------------------+---------------+----+-----+-----+-----+-----+-----+-----+----+----+----+----+----+----+----+----+----+---+---+---+---+---+---+---+---+---+---+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+\n",
       "|Zürich Tiefenbrunnen|85:11:19639:001|   2|    0|    0|    0|    0|    0|    0|   0|   0|   0|   0|   0|   0|   0|   0|   0|  0| 65| 10|  5|  2|  2|  2|  1|  0|  0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|    0|    0|    0|    0|    0|    0|    0|    0|    0|    0|    0|\n",
       "|         Zürich Enge|85:11:18267:001|   0|    0|    0|    0|    0|    0|    0|   0|   0|   0|   0|   0|   0|   0|   0|   0|  5|  1|  1|  0|  0|  0|  0|  0|  0|  0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|    0|    0|    0|    0|    0|    0|    0|    0|    0|    0|    0|\n",
       "|           Zürich HB|85:11:30992:009|   0|    0|    0|    0|    0|    0|    0|   0|   0|   0|   0|   0|   0|   0|   0|   9|  3|  1|  0|  0|  0|  0|  0|  1|  0|  0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|    0|    0|    0|    0|    0|    0|    0|    0|    0|    0|    0|\n",
       "|   Zürich Altstetten|85:11:19978:001|   0|    0|    0|    0|    0|    0|    0|   0|   0|   0|   0|   0|   0|   0|   0|   2| 70|  8|  2|  0|  1|  1|  0|  0|  0|  0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|    0|    0|    0|    0|    0|    0|    0|    0|    0|    0|    0|\n",
       "|           Zürich HB|85:11:18873:001|   0|    0|    0|    0|    0|    0|    0|   0|   0|   0|   0|   0|   0|   0|   0|   0| 20| 22| 24|  7|  7|  1|  0|  0|  1|  0|   0|   0|   1|   0|   0|   0|   1|   0|   1|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|    0|    0|    0|    0|    0|    0|    0|    0|    0|    0|    0|\n",
       "+--------------------+---------------+----+-----+-----+-----+-----+-----+-----+----+----+----+----+----+----+----+----+----+---+---+---+---+---+---+---+---+---+---+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+\n",
       "only showing top 5 rows"
      ]
     }
    ],
    "source": [
     "sbb_filt.groupBy('station', 'trip_id').pivot(\"DiffInMinutes\").count()\\\n",
     "        .na.fill(0)\\\n",
     "        .show(5)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "As an addition to this distribution, we can set up lower / upper bound to constrain the distribution to a specific window of interest. We do not really care about train being ahead, so we put them all in -1 column index, And we look at delays until 30 minutes only."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 39,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "+------------------+---------------+----+---+---+---+---+---+---+---+---+---+---+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+\n",
       "|           station|        trip_id|-1.0|0.0|1.0|2.0|3.0|4.0|5.0|6.0|7.0|8.0|9.0|10.0|11.0|12.0|13.0|14.0|15.0|16.0|17.0|18.0|19.0|20.0|21.0|22.0|23.0|24.0|25.0|26.0|27.0|28.0|29.0|30.0|\n",
       "+------------------+---------------+----+---+---+---+---+---+---+---+---+---+---+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+\n",
       "|         Zürich HB|85:11:30992:009|   9|  3|  1|  0|  0|  0|  0|  0|  1|  0|  0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
       "|         Zürich HB|85:11:18873:001|   0| 20| 22| 24|  7|  7|  1|  0|  0|  1|  0|   0|   0|   1|   0|   0|   0|   1|   0|   1|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
       "|   Zürich Oerlikon|85:11:20438:002|   0|  0| 27| 50|  8|  4|  0|  0|  0|  0|  0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
       "|Zürich Wollishofen|85:11:18822:001|   0|  2|  2|  1|  0|  0|  0|  0|  0|  0|  0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
       "|  Zürich Flughafen| 85:11:2270:001|   9| 46| 30|  2|  1|  1|  0|  0|  0|  0|  0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
       "+------------------+---------------+----+---+---+---+---+---+---+---+---+---+---+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+\n",
       "only showing top 5 rows"
      ]
     }
    ],
    "source": [
     "lower_bound = -1.0\n",
     "upped_bound = +30.0\n",
     "\n",
     "sbb_bounded = sbb_filt.withColumn('DiffInMinutes_bounded1',\\\n",
     "                                    greatest(col('DiffInMinutes'), lit(lower_bound) ))\\\n",
     "                      .withColumn('DiffInMinutes_bounded2',\\\n",
     "                                    least(col('DiffInMinutes_bounded1'), lit(upped_bound) ))\n",
     "\n",
     "sbb_bounded.groupBy('station', 'trip_id').pivot(\"DiffInMinutes_bounded2\").count()\\\n",
     "        .na.fill(0)\\\n",
     "        .show(5)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "### Work from translation tables \n",
     "\n",
     "We will use data generated in `match_datasets.ipynb`. We begin by looking at all trip_id that are found in both dataset with at least 5 stations in common.\n",
     "\n",
     "Our goal is to find a match in sbb dataset for all _timetable_ trips (and not the other way around). So we will focus on getting this assymetrical correspondance table. \n",
     "\n",
     "When we find a clear one-one match, we will mark them as _resolved_, when there is a one-to-many relation, we will call it _partly_resolved_ and if we cannot find a sbb trip that correspond to a timetable trip_id, we will call it _fail_to_resolve_. \n",
     "\n",
     "These labels will be used to differentiate 3 different ways to compute probabilities :\n",
     "- __One-to-one__ we find a clear match : we use distribution of delays on weekdays for a given trip/station_id based on all past sbb data. \n",
     "- __One-to-many__ we find multiple match :\n",
     "    - First we double check the matches, if we have the same type of transportation for example.\n",
     "    - If they seem to be correct, we can merge the trips from sbb and get the merged distribution of their delays.\n",
     "- __One-to-none__ we find no match : then we get the distribution of delays for similar transportation types, at similar hour (in a window), during weekdays of sbb dataset.\n",
     "    - Alternative : Try to find the best match and use only the closest location/time to infer a given distribution.\n",
     "    - Alternative 2 : use k-nearest neighbors in terms of location/time."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "243152\n",
       "+------------------------+----------------------+-----+\n",
       "|trip_id                 |fahrt_bezeichner      |count|\n",
       "+------------------------+----------------------+-----+\n",
       "|241.TA.26-14-j19-1.43.H |85:11:19435:001       |13   |\n",
       "|1419.TA.26-8-C-j19-1.8.R|85:3849:169172-07008-1|23   |\n",
       "|1015.TA.26-4-j19-1.25.H |85:3849:49891-03002-1 |7    |\n",
       "|1955.TA.26-13-j19-1.24.H|85:3849:89261-02013-1 |5    |\n",
       "|1217.TA.26-72-j19-1.6.R |85:849:55624-25033-1  |7    |\n",
       "+------------------------+----------------------+-----+\n",
       "only showing top 5 rows"
      ]
     }
    ],
    "source": [
     "joined_trip_atL5 = spark.read.csv('data/lgpt_guys/joined_trip_atL5.csv', header = True)\n",
     "print joined_trip_atL5.count()\n",
     "joined_trip_atL5.show(5, False)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "31103"
      ]
     }
    ],
    "source": [
     "joined_trip_atL5.select('fahrt_bezeichner').distinct().count()"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "We also use the subset of sbb data (we use the filtered data `sbb_filt` made at the top of the notebook)."
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "We can begin by assembling sbb data set with translation table `joined_trip_atL5` "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "10848628\n",
+      "+----------------+-----------------+----------------+----------------+-------------------+-------------------+-------+\n",
+      "|fahrt_bezeichner|haltestellen_name|    ankunftszeit|    abfahrtszeit|        an_prognose|        ab_prognose|stop_id|\n",
+      "+----------------+-----------------+----------------+----------------+-------------------+-------------------+-------+\n",
+      "|    85:11:10:002|        Zürich HB|12.10.2018 21:51|                |12.10.2018 21:51:50|                   |8503000|\n",
+      "| 85:11:10293:004|        Zürich HB|                |13.10.2018 00:25|                   |13.10.2018 00:26:08|8503000|\n",
+      "| 85:11:10293:004| Zürich Flughafen|13.10.2018 00:34|13.10.2018 00:35|13.10.2018 00:35:27|13.10.2018 00:36:44|8503016|\n",
+      "+----------------+-----------------+----------------+----------------+-------------------+-------------------+-------+\n",
+      "only showing top 3 rows"
+     ]
+    }
+   ],
+   "source": [
+    "username = 'acoudray'\n",
+    "sbb_filt = spark.read.orc(\"/user/{}/sbb_filt_forDelays_GeschaetzAndReal.orc\".format(username))\n",
+    "print(sbb_filt.count())\n",
+    "sbb_filt.show(3)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "652832165\n",
+      "16474877\n",
       "+----------------+-----------------+----------------+----------------+-------------------+-------------------+-------+-------+-----+\n",
       "|fahrt_bezeichner|haltestellen_name|ankunftszeit    |abfahrtszeit    |an_prognose        |ab_prognose        |stop_id|trip_id|count|\n",
       "+----------------+-----------------+----------------+----------------+-------------------+-------------------+-------+-------+-----+\n",
-      "|85:11:10:002    |Zürich HB        |03.09.2018 21:51|                |03.09.2018 21:53:40|                   |8503000|null   |null |\n",
-      "|85:11:11:001    |Zürich HB        |                |03.09.2018 06:09|                   |03.09.2018 06:10:22|8503000|null   |null |\n",
-      "|85:11:12:001    |Zürich HB        |03.09.2018 10:51|                |03.09.2018 10:51:28|                   |8503000|null   |null |\n",
-      "|85:11:1251:003  |Zürich HB        |03.09.2018 07:00|                |03.09.2018 07:00:01|                   |8503000|null   |null |\n",
-      "|85:11:1252:001  |Zürich HB        |03.09.2018 21:23|03.09.2018 21:36|03.09.2018 21:24:55|03.09.2018 21:36:57|8503000|null   |null |\n",
-      "|85:11:1255:001  |Zürich HB        |03.09.2018 08:26|03.09.2018 08:37|03.09.2018 08:28:06|03.09.2018 08:39:07|8503000|null   |null |\n",
-      "|85:11:1256:003  |Zürich HB        |03.09.2018 17:53|                |03.09.2018 17:55:21|                   |8503000|null   |null |\n",
-      "|85:11:1260:004  |Zürich HB        |                |03.09.2018 21:00|                   |03.09.2018 21:01:13|8503000|null   |null |\n",
-      "|85:11:1271:001  |Zürich HB        |03.09.2018 10:00|03.09.2018 10:07|03.09.2018 09:59:07|03.09.2018 10:08:45|8503000|null   |null |\n",
-      "|85:11:13:001    |Zürich HB        |                |03.09.2018 07:09|                   |03.09.2018 07:11:14|8503000|null   |null |\n",
+      "|85:11:10:002    |Zürich HB        |12.10.2018 21:51|                |12.10.2018 21:51:50|                   |8503000|null   |null |\n",
+      "|85:11:10293:004 |Zürich HB        |                |13.10.2018 00:25|                   |13.10.2018 00:26:08|8503000|null   |null |\n",
+      "|85:11:10293:004 |Zürich Flughafen |13.10.2018 00:34|13.10.2018 00:35|13.10.2018 00:35:27|13.10.2018 00:36:44|8503016|null   |null |\n",
+      "|85:11:10536:004 |Zürich HB        |                |12.10.2018 20:03|                   |12.10.2018 20:04:20|8503000|null   |null |\n",
+      "|85:11:10537:006 |Zürich HB        |12.10.2018 21:59|                |12.10.2018 22:01:43|                   |8503000|null   |null |\n",
+      "|85:11:10538:004 |Zürich HB        |                |12.10.2018 21:03|                   |12.10.2018 21:04:42|8503000|null   |null |\n",
+      "|85:11:10539:005 |Zürich HB        |12.10.2018 22:59|                |12.10.2018 23:00:10|                   |8503000|null   |null |\n",
+      "|85:11:10540:004 |Zürich HB        |                |12.10.2018 22:03|                   |12.10.2018 22:06:29|8503000|null   |null |\n",
+      "|85:11:10734:007 |Zürich Flughafen |12.10.2018 20:16|12.10.2018 20:18|12.10.2018 20:15:27|12.10.2018 20:18:39|8503016|null   |null |\n",
+      "|85:11:10734:007 |Zürich HB        |12.10.2018 20:27|12.10.2018 20:32|12.10.2018 20:26:44|12.10.2018 20:33:02|8503000|null   |null |\n",
       "+----------------+-----------------+----------------+----------------+-------------------+-------------------+-------+-------+-----+\n",
       "only showing top 10 rows"
      ]
     }
    ],
    "source": [
     "joined_sbb = sbb_filt.join(joined_trip_atL5, on = ['fahrt_bezeichner'], how = 'left_outer')\n",
     "\n",
     "print joined_sbb.count()\n",
     "joined_sbb.show(10,False)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "1107043"
+      "46399"
      ]
     }
    ],
    "source": [
     "joined_sbb.select(\"fahrt_bezeichner\", \"trip_id\").distinct().count()"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "The reference table we will use is the `stop_times` tables containing trip_id and stop_id. As a next step, we will put them in the same order raptor will read them."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "247920\n",
-      "+-----------+--------------------+-------+\n",
-      "|stop_id_raw|trip_id             |stop_id|\n",
-      "+-----------+--------------------+-------+\n",
-      "|8572747    |1.TA.1-231-j19-1.1.H|8572747|\n",
-      "|8573721    |1.TA.1-231-j19-1.1.H|8573721|\n",
-      "|8503598    |1.TA.1-231-j19-1.1.H|8503598|\n",
-      "+-----------+--------------------+-------+\n",
-      "only showing top 3 rows"
+      "+--------------------+-------+------------+--------------+-------------+-----------+-------------+--------------+------------+------------+\n",
+      "|             trip_id|stop_id|arrival_time|departure_time|stop_sequence|pickup_type|drop_off_type|hour_departure|    route_id|direction_id|\n",
+      "+--------------------+-------+------------+--------------+-------------+-----------+-------------+--------------+------------+------------+\n",
+      "|666.TA.26-4-j19-1...|8576182|    07:02:00|      07:02:00|            1|          0|            0|           7.0|  26-4-j19-1|           1|\n",
+      "|243.TA.26-311-j19...|8590834|    07:16:00|      07:16:00|            1|          0|            0|           7.0|26-311-j19-1|           1|\n",
+      "+--------------------+-------+------------+--------------+-------------+-----------+-------------+--------------+------------+------------+\n",
+      "only showing top 2 rows"
      ]
     }
    ],
    "source": [
     "stop_times_curated = spark.read.csv('data/lgpt_guys/stop_times_curated.csv', header = True)\n",
+    "stop_times_curated.show(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "250777\n",
+      "+-----------+-----------------------+-------+\n",
+      "|stop_id_raw|trip_id                |stop_id|\n",
+      "+-----------+-----------------------+-------+\n",
+      "|8576182    |666.TA.26-4-j19-1.20.R |8576182|\n",
+      "|8590834    |243.TA.26-311-j19-1.3.R|8590834|\n",
+      "|8591349    |406.TA.26-62-j19-1.3.R |8591349|\n",
+      "+-----------+-----------------------+-------+\n",
+      "only showing top 3 rows"
+     ]
+    }
+   ],
+   "source": [
     "# We use only first 7 characters of stop_id to remove special cases\n",
     "stop_times_curated = stop_times_curated.select(col('stop_id').alias('stop_id_raw'), \n",
     "                                               'trip_id')\\\n",
     "                       .withColumn('stop_id',col('stop_id_raw').substr(1, 7))\n",
     "\n",
     "print stop_times_curated.count()\n",
     "stop_times_curated.show(3, False)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "22807"
+      "19800"
      ]
     }
    ],
    "source": [
     "stop_times_curated.select('trip_id').distinct().count()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "341372874\n",
-      "+---------------------+-------+-----+--------------------+----------------+----------------+-------------------+-------------------+\n",
-      "|trip_id              |stop_id|count|fahrt_bezeichner    |ankunftszeit    |abfahrtszeit    |an_prognose        |ab_prognose        |\n",
-      "+---------------------+-------+-----+--------------------+----------------+----------------+-------------------+-------------------+\n",
-      "|1.TA.26-163-j19-1.1.R|8590688|5    |85:849:62887-01162-1|11.12.2019 08:02|11.12.2019 08:02|11.12.2019 08:03:47|11.12.2019 08:03:59|\n",
-      "|1.TA.26-163-j19-1.1.R|8590688|5    |85:849:62887-01162-1|21.05.2019 08:02|21.05.2019 08:02|21.05.2019 08:08:11|21.05.2019 08:08:23|\n",
-      "|1.TA.26-163-j19-1.1.R|8590688|11   |85:849:62934-01162-1|11.12.2019 19:54|11.12.2019 19:54|11.12.2019 19:54:33|11.12.2019 19:54:39|\n",
-      "|1.TA.26-163-j19-1.1.R|8590688|11   |85:849:62934-01162-1|21.05.2019 19:54|21.05.2019 19:54|21.05.2019 19:54:52|21.05.2019 19:54:58|\n",
-      "|1.TA.26-163-j19-1.1.R|8590688|5    |85:849:62887-01162-1|14.11.2019 08:02|14.11.2019 08:02|14.11.2019 08:06:27|14.11.2019 08:06:39|\n",
-      "|1.TA.26-163-j19-1.1.R|8590688|5    |85:849:62887-01162-1|16.04.2019 08:02|16.04.2019 08:02|16.04.2019 08:06:37|16.04.2019 08:06:49|\n",
-      "|1.TA.26-163-j19-1.1.R|8590688|11   |85:849:62934-01162-1|14.11.2019 19:54|14.11.2019 19:54|14.11.2019 19:54:35|14.11.2019 19:54:41|\n",
-      "|1.TA.26-163-j19-1.1.R|8590688|11   |85:849:62934-01162-1|16.04.2019 19:54|16.04.2019 19:54|16.04.2019 19:55:52|16.04.2019 19:55:58|\n",
-      "|1.TA.26-163-j19-1.1.R|8590688|5    |85:849:62887-01162-1|13.12.2019 08:02|13.12.2019 08:02|13.12.2019 08:04:30|13.12.2019 08:04:42|\n",
-      "|1.TA.26-163-j19-1.1.R|8590688|5    |85:849:62887-01162-1|15.04.2019 08:02|15.04.2019 08:02|15.04.2019 08:06:39|15.04.2019 08:06:51|\n",
-      "+---------------------+-------+-----+--------------------+----------------+----------------+-------------------+-------------------+\n",
+      "9478785\n",
+      "+-------------------------+-------+-----+----------------+------------+------------+-----------+-----------+\n",
+      "|trip_id                  |stop_id|count|fahrt_bezeichner|ankunftszeit|abfahrtszeit|an_prognose|ab_prognose|\n",
+      "+-------------------------+-------+-----+----------------+------------+------------+-----------+-----------+\n",
+      "|1.TA.26-89-j19-1.1.R     |8591209|null |null            |null        |null        |null       |null       |\n",
+      "|10.TA.1-305-j19-1.1.R    |8587018|null |null            |null        |null        |null       |null       |\n",
+      "|10.TA.26-69-j19-1.2.H    |8591122|null |null            |null        |null        |null       |null       |\n",
+      "|10.TA.26-845-j19-1.2.H   |8580879|null |null            |null        |null        |null       |null       |\n",
+      "|10.TA.26-918-j19-1.1.R   |8590701|null |null            |null        |null        |null       |null       |\n",
+      "|10.TA.79-485-j19-1.1.R   |8590461|null |null            |null        |null        |null       |null       |\n",
+      "|100.TA.26-748-j19-1.1.R  |8590543|null |null            |null        |null        |null       |null       |\n",
+      "|1001.TA.26-70-A-j19-1.5.H|8591106|null |null            |null        |null        |null       |null       |\n",
+      "|1005.TA.26-70-A-j19-1.5.H|8591410|null |null            |null        |null        |null       |null       |\n",
+      "|1008.TA.26-142-j19-1.2.R |8590830|null |null            |null        |null        |null       |null       |\n",
+      "+-------------------------+-------+-----+----------------+------------+------------+-----------+-----------+\n",
       "only showing top 10 rows"
      ]
     }
    ],
    "source": [
     "stop_times_join = stop_times_curated.join(joined_sbb, on=['trip_id', 'stop_id'], \n",
     "                              how='left_outer')\\\n",
     "                   .select('trip_id', 'stop_id', 'count',\n",
     "                           'fahrt_bezeichner', 'ankunftszeit', 'abfahrtszeit',\n",
     "                           'an_prognose', 'ab_prognose')\n",
     "\n",
     "print stop_times_join.count()\n",
     "stop_times_join.show(10, False)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "We then compute arrival delays using the following approach : \n",
     "- arrival_true ( = `an_prognose`) - arrival_expected ( = `ankunftszeit`). Train being late have a positive delay and trains being ahead of schedule a negative one."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "+-------+------------------------+-------------------+-------------+-------------+-------+\n",
       "|stop_id|trip_id                 |arrival_true       |DiffInSeconds|DiffInMinutes|weekday|\n",
       "+-------+------------------------+-------------------+-------------+-------------+-------+\n",
-      "|8503006|419.TA.26-2-j19-1.164.H |2018-09-03 06:10:53|53           |0            |Mon    |\n",
-      "|8503000|419.TA.26-2-j19-1.164.H |2018-09-03 06:16:22|82           |1            |Mon    |\n",
-      "|8503011|419.TA.26-2-j19-1.164.H |2018-09-03 06:19:51|51           |0            |Mon    |\n",
-      "|8503010|419.TA.26-2-j19-1.164.H |2018-09-03 06:21:58|-2           |0            |Mon    |\n",
-      "|8503202|419.TA.26-2-j19-1.164.H |2018-09-03 06:30:11|11           |0            |Mon    |\n",
-      "|8503204|419.TA.26-2-j19-1.164.H |2018-09-03 06:34:44|-16          |0            |Mon    |\n",
-      "|8503204|214.TA.26-24-j19-1.121.R|2018-09-03 06:31:17|497          |8            |Mon    |\n",
-      "|8503204|74.TA.26-2-j19-1.9.R    |2018-09-03 06:31:17|497          |8            |Mon    |\n",
-      "|8503202|214.TA.26-24-j19-1.121.R|2018-09-03 06:35:52|472          |7            |Mon    |\n",
-      "|8503202|74.TA.26-2-j19-1.9.R    |2018-09-03 06:35:52|472          |7            |Mon    |\n",
+      "|8503006|419.TA.26-2-j19-1.164.H |2018-10-12 06:10:43|43           |0            |Fri    |\n",
+      "|8503000|419.TA.26-2-j19-1.164.H |2018-10-12 06:15:56|56           |0            |Fri    |\n",
+      "|8503011|419.TA.26-2-j19-1.164.H |2018-10-12 06:19:45|45           |0            |Fri    |\n",
+      "|8503010|419.TA.26-2-j19-1.164.H |2018-10-12 06:21:35|-25          |0            |Fri    |\n",
+      "|8503202|419.TA.26-2-j19-1.164.H |2018-10-12 06:29:28|-32          |0            |Fri    |\n",
+      "|8503204|419.TA.26-2-j19-1.164.H |2018-10-12 06:34:42|-18          |0            |Fri    |\n",
+      "|8503204|214.TA.26-24-j19-1.121.R|2018-10-12 06:23:17|17           |0            |Fri    |\n",
+      "|8503204|74.TA.26-2-j19-1.9.R    |2018-10-12 06:23:17|17           |0            |Fri    |\n",
+      "|8503202|214.TA.26-24-j19-1.121.R|2018-10-12 06:27:58|-2           |0            |Fri    |\n",
+      "|8503202|74.TA.26-2-j19-1.9.R    |2018-10-12 06:27:58|-2           |0            |Fri    |\n",
       "+-------+------------------------+-------------------+-------------+-------------+-------+\n",
       "only showing top 10 rows"
      ]
     }
    ],
    "source": [
     "stop_times_diff = stop_times_join.select( col(\"an_prognose\").alias(\"arrival_true\"),\\\n",
     "                              col(\"ankunftszeit\").alias(\"arrival_expected\"),\\\n",
     "                              'trip_id', 'stop_id')\\\n",
     "              .withColumn('arrival_true',to_timestamp(col('arrival_true'),\\\n",
     "                                                          format='dd.MM.yyyy HH:mm:ss'))\\\n",
     "              .withColumn('arrival_expected',to_timestamp(col('arrival_expected'),\\\n",
     "                                                           format='dd.MM.yyyy HH:mm'))\\\n",
     "              .withColumn('DiffInSeconds',col('arrival_true').cast(LongType()) - col('arrival_expected').cast(LongType()))\\\n",
     "              .withColumn('DiffInMinutes',(col('DiffInSeconds')/60).cast('integer'))\\\n",
     "              .select(\"stop_id\", \"trip_id\", \"arrival_true\", \"DiffInSeconds\", \"DiffInMinutes\",\\\n",
     "                        date_format('arrival_expected', 'E').alias('weekday'))\n",
     "\n",
     "# Remove Saturday and Sunday weekdays from table - show\n",
     "stop_times_diff = stop_times_diff.filter( (stop_times_diff.weekday != \"Sun\") & (stop_times_diff.weekday != \"Sat\") )\n",
     "stop_times_diff.show(10, False)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "+-------+-------------------------+---+----+----+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
-      "|stop_id|trip_id                  |-1 |0   |1   |2  |3  |4  |5  |6  |7  |8  |9  |10 |11 |12 |13 |14 |15 |16 |17 |18 |19 |20 |21 |22 |23 |24 |25 |26 |27 |28 |29 |30 |\n",
-      "+-------+-------------------------+---+----+----+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
-      "|8503016|460.TA.26-24-j19-1.220.R |15 |73  |114 |64 |18 |6  |4  |0  |1  |1  |0  |1  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
-      "|8591079|313.TA.26-2-A-j19-1.1.H  |15 |2240|683 |204|37 |6  |8  |5  |2  |1  |0  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |2  |0  |0  |0  |0  |3  |\n",
-      "|8576195|1458.TA.26-2-A-j19-1.24.R|15 |2269|825 |316|110|58 |33 |13 |10 |7  |7  |0  |1  |1  |0  |0  |2  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |4  |\n",
-      "|8591299|1361.TA.26-2-A-j19-1.24.R|15 |1551|550 |109|30 |14 |13 |3  |4  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |1  |0  |0  |0  |0  |0  |1  |\n",
-      "|8591305|1384.TA.26-7-B-j19-1.6.R |6  |1557|405 |153|110|59 |22 |13 |8  |2  |0  |1  |1  |0  |1  |0  |1  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
-      "|8591298|2509.TA.26-7-B-j19-1.17.H|21 |2504|946 |267|82 |20 |10 |6  |3  |7  |3  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |1  |\n",
-      "|8591174|356.TA.26-15-A-j19-1.5.R |8  |2070|851 |242|84 |21 |5  |6  |5  |0  |1  |1  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |1  |0  |0  |0  |0  |0  |5  |\n",
-      "|8591276|926.TA.26-14-A-j19-1.10.R|5  |3024|1772|538|147|48 |15 |9  |3  |1  |0  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |1  |0  |1  |0  |0  |0  |0  |0  |0  |\n",
-      "|8530813|540.TA.26-18-j19-1.12.H  |805|76  |96  |32 |16 |5  |4  |0  |1  |1  |2  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
-      "|8591299|1362.TA.26-13-j19-1.20.R |40 |1986|1158|482|185|100|41 |19 |10 |5  |3  |3  |1  |0  |0  |1  |1  |0  |0  |0  |0  |0  |0  |0  |1  |0  |0  |0  |0  |0  |0  |2  |\n",
-      "|8591306|1972.TA.26-13-j19-1.24.H |31 |3243|1130|410|136|48 |9  |9  |6  |5  |2  |0  |0  |2  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |2  |\n",
-      "|8576152|36.TA.26-732-j19-1.2.H   |0  |185 |160 |83 |30 |17 |7  |2  |0  |1  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
-      "|8590550|243.TA.26-752-j19-1.4.R  |0  |607 |492 |191|30 |11 |1  |0  |0  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
-      "|8591354|1098.TA.26-768-j19-1.2.R |0  |128 |127 |60 |10 |9  |3  |3  |2  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
-      "|8580874|69.TA.26-817-j19-1.1.H   |0  |16  |52  |40 |35 |22 |11 |4  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
-      "|8591371|1313.TA.26-46-j19-1.9.H  |0  |702 |319 |210|118|39 |34 |16 |8  |3  |1  |1  |2  |0  |2  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
-      "|8591293|290.TA.26-62-j19-1.2.H   |1  |1120|1084|478|179|59 |23 |17 |4  |3  |1  |0  |0  |2  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |8  |\n",
-      "|8580522|55.TA.26-83-j19-1.1.R    |42 |2031|1180|488|157|62 |47 |28 |24 |20 |6  |5  |5  |4  |4  |2  |0  |5  |4  |3  |2  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |1  |\n",
-      "|8591163|1272.TA.26-80-j19-1.8.R  |1  |450 |198 |99 |46 |15 |2  |4  |0  |0  |0  |0  |0  |0  |1  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
-      "|8591182|500.TA.26-89-j19-1.6.H   |1  |415 |297 |233|128|75 |31 |16 |11 |5  |5  |2  |2  |0  |0  |0  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
-      "+-------+-------------------------+---+----+----+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
+      "+-------+-------------------------+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
+      "|stop_id|trip_id                  |-1 |0  |1  |2  |3  |4  |5  |6  |7  |8  |9  |10 |11 |12 |13 |14 |15 |16 |17 |18 |19 |20 |21 |22 |23 |24 |25 |26 |27 |28 |29 |30 |\n",
+      "+-------+-------------------------+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
+      "|8503020|45.TA.26-7-A-j19-1.12.H  |0  |537|107|38 |13 |2  |3  |4  |1  |1  |1  |0  |1  |1  |0  |1  |1  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8594307|44.TA.1-11-B-j19-1.2.H   |0  |1  |4  |1  |2  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503125|59.TA.26-5-A-j19-1.28.R  |0  |578|179|30 |12 |5  |2  |0  |1  |1  |0  |1  |1  |0  |0  |2  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8590275|501.TA.1-2-A-j19-1.15.R  |0  |23 |28 |9  |4  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503000|147.TA.26-15-j19-1.41.H  |0  |271|114|20 |7  |2  |1  |0  |0  |0  |0  |0  |0  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503203|590.TA.26-8-A-j19-1.353.H|0  |463|648|340|106|34 |20 |8  |5  |3  |6  |3  |3  |7  |3  |1  |1  |1  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8502208|432.TA.26-24-j19-1.220.R |0  |86 |40 |10 |3  |2  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503305|201.TA.26-24-j19-1.121.R |0  |184|60 |22 |7  |5  |2  |0  |1  |0  |0  |0  |0  |1  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8590279|136.TA.1-4-B-j19-1.10.H  |0  |4  |4  |3  |2  |2  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503003|258.TA.26-16-A-j19-1.93.H|85 |913|359|167|53 |34 |20 |10 |4  |5  |3  |2  |0  |0  |1  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503011|571.TA.26-8-A-j19-1.347.H|0  |590|457|116|40 |16 |4  |7  |1  |1  |2  |1  |1  |1  |0  |0  |0  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503020|389.TA.26-7-A-j19-1.108.R|0  |243|150|19 |1  |1  |0  |1  |0  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8517376|197.TA.1-17-A-j19-1.16.R |0  |274|499|212|101|34 |4  |2  |1  |0  |0  |1  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503006|377.TA.26-7-A-j19-1.108.R|0  |541|223|58 |9  |2  |2  |1  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503104|135.TA.26-6-A-j19-1.32.R |0  |394|165|63 |30 |10 |5  |5  |2  |2  |1  |1  |0  |1  |1  |0  |0  |0  |0  |0  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8502187|187.TA.1-17-A-j19-1.16.R |1  |479|71 |6  |4  |1  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503020|105.TA.26-5-A-j19-1.37.R |0  |443|226|98 |35 |14 |5  |2  |0  |2  |1  |0  |2  |1  |2  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8502221|54.TA.26-5-A-j19-1.28.R  |152|492|116|41 |14 |6  |8  |2  |1  |0  |0  |1  |0  |1  |2  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8502273|101.TA.1-17-A-j19-1.9.R  |2  |507|330|205|95 |25 |4  |2  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503052|238.TA.26-10-B-j19-1.10.R|0  |260|82 |47 |21 |12 |6  |3  |3  |1  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "+-------+-------------------------+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
       "only showing top 20 rows"
      ]
     }
    ],
    "source": [
     "# we bound distribution to this \n",
     "lower_bound = -1\n",
     "upped_bound = +30\n",
     "\n",
     "stop_times_bounded = stop_times_diff.withColumn('DiffInMinutes_bounded1',\\\n",
     "                                        greatest(col('DiffInMinutes'), lit(lower_bound) ))\\\n",
     "                                    .withColumn('DiffInMinutes_bounded2',\\\n",
     "                                        least(col('DiffInMinutes_bounded1'), lit(upped_bound) ))\n",
     "\n",
     "stop_times_distribution = stop_times_bounded.groupBy('stop_id', 'trip_id')\\\n",
     "                                            .pivot(\"DiffInMinutes_bounded2\").count()\\\n",
     "                                            .na.fill(0)\n",
     "\n",
     "stop_times_distribution.show(20, False)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 38,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "223579"
+      "12309"
      ]
     }
    ],
    "source": [
     "stop_times_distribution.count()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 27,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     }
    ],
    "source": [
-    "stop_times_distribution.write.csv('data/lgpt_guys/distribution_1to1match.csv', \\\n",
+    "stop_times_distribution.write.csv('data/lgpt_guys/distribution_geschaetzAndReal.csv', \\\n",
     "                                  header = True, mode=\"overwrite\")"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "### Analysing matches found \n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 28,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "+--------------------+-------+------------+--------------+-------------+-----------+-------------+--------+-----------+----------+--------------------+------------+\n",
-      "|             trip_id|stop_id|arrival_time|departure_time|stop_sequence|pickup_type|drop_off_type|stop_int|   route_id|sequence_1|              trip_1|   route_int|\n",
-      "+--------------------+-------+------------+--------------+-------------+-----------+-------------+--------+-----------+----------+--------------------+------------+\n",
-      "|1.TA.1-231-j19-1.1.H|8572747|    09:37:00|      09:37:00|            1|          0|            0|     500|1-231-j19-1|      10.0|1.TA.1-231-j19-1.1.H|592705486850|\n",
-      "|1.TA.1-231-j19-1.1.H|8573721|    09:50:00|      09:50:00|           10|          0|            0|     599|1-231-j19-1|      11.0|1.TA.1-231-j19-1.1.H|592705486850|\n",
-      "|1.TA.1-231-j19-1.1.H|8503598|    09:53:00|      09:53:00|           11|          0|            0|     401|1-231-j19-1|      12.0|1.TA.1-231-j19-1.1.H|592705486850|\n",
-      "|1.TA.1-231-j19-1.1.H|8573720|    09:55:00|      09:59:00|           12|          0|            0|     598|1-231-j19-1|      13.0|1.TA.1-231-j19-1.1.H|592705486850|\n",
-      "|1.TA.1-231-j19-1.1.H|8503598|    10:00:00|      10:00:00|           13|          0|            0|     401|1-231-j19-1|      14.0|1.TA.1-231-j19-1.1.H|592705486850|\n",
-      "+--------------------+-------+------------+--------------+-------------+-----------+-------------+--------+-----------+----------+--------------------+------------+\n",
+      "+--------------------+------------+------------+--------------+-------------+-----------+-------------+--------------+------------+------------+\n",
+      "|             trip_id|     stop_id|arrival_time|departure_time|stop_sequence|pickup_type|drop_off_type|hour_departure|    route_id|direction_id|\n",
+      "+--------------------+------------+------------+--------------+-------------+-----------+-------------+--------------+------------+------------+\n",
+      "|666.TA.26-4-j19-1...|     8576182|    07:02:00|      07:02:00|            1|          0|            0|           7.0|  26-4-j19-1|           1|\n",
+      "|243.TA.26-311-j19...|     8590834|    07:16:00|      07:16:00|            1|          0|            0|           7.0|26-311-j19-1|           1|\n",
+      "|406.TA.26-62-j19-...|     8591349|    07:24:00|      07:24:00|            1|          0|            0|           7.0| 26-62-j19-1|           1|\n",
+      "|62.TA.57-2-Y-j19-...|8503000:0:13|    07:34:00|      07:34:00|            1|          0|            0|           7.0|57-2-Y-j19-1|           0|\n",
+      "|1179.TA.26-5-B-j1...|     8591245|    07:36:00|      07:36:00|            1|          0|            0|           7.0|26-5-B-j19-1|           1|\n",
+      "+--------------------+------------+------------+--------------+-------------+-----------+-------------+--------------+------------+------------+\n",
       "only showing top 5 rows"
      ]
     }
    ],
    "source": [
     "stop_times_curated = spark.read.csv('data/lgpt_guys/stop_times_curated.csv', header = True)\n",
     "stop_times_curated.show(5)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 29,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "247920\n",
-      "+-----------+--------------------+-------+\n",
-      "|stop_id_raw|trip_id             |stop_id|\n",
-      "+-----------+--------------------+-------+\n",
-      "|8572747    |1.TA.1-231-j19-1.1.H|8572747|\n",
-      "|8573721    |1.TA.1-231-j19-1.1.H|8573721|\n",
-      "|8503598    |1.TA.1-231-j19-1.1.H|8503598|\n",
-      "+-----------+--------------------+-------+\n",
+      "250777\n",
+      "+-----------+-----------------------+-------+\n",
+      "|stop_id_raw|trip_id                |stop_id|\n",
+      "+-----------+-----------------------+-------+\n",
+      "|8576182    |666.TA.26-4-j19-1.20.R |8576182|\n",
+      "|8590834    |243.TA.26-311-j19-1.3.R|8590834|\n",
+      "|8591349    |406.TA.26-62-j19-1.3.R |8591349|\n",
+      "+-----------+-----------------------+-------+\n",
       "only showing top 3 rows"
      ]
     }
    ],
    "source": [
     "# We use only first 7 characters of stop_id to remove special cases\n",
     "stop_times_curated = stop_times_curated.select(col('stop_id').alias('stop_id_raw'), \n",
     "                                               'trip_id')\\\n",
     "                       .withColumn('stop_id',col('stop_id_raw').substr(1, 7))\n",
     "\n",
     "print stop_times_curated.count()\n",
     "stop_times_curated.show(3, False)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 76,
+   "execution_count": 30,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "223579\n",
-      "+-------+-------------------------+---+----+----+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
-      "|stop_id|trip_id                  |-1 |0   |1   |2  |3  |4  |5  |6  |7  |8  |9  |10 |11 |12 |13 |14 |15 |16 |17 |18 |19 |20 |21 |22 |23 |24 |25 |26 |27 |28 |29 |30 |\n",
-      "+-------+-------------------------+---+----+----+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
-      "|8591415|285.TA.26-5-B-j19-1.3.H  |1  |1115|429 |261|101|29 |10 |5  |3  |3  |4  |3  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
-      "|8591298|777.TA.26-15-A-j19-1.6.H |6  |977 |599 |240|89 |30 |19 |7  |2  |3  |2  |2  |0  |0  |1  |1  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |1  |0  |0  |9  |\n",
-      "|8591093|1407.TA.26-2-A-j19-1.24.R|9  |1484|568 |174|40 |16 |8  |7  |2  |0  |0  |3  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |1  |0  |0  |0  |\n",
-      "|8591220|305.TA.26-9-B-j19-1.1.R  |26 |1865|742 |214|57 |14 |6  |3  |2  |3  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |2  |\n",
-      "|8591368|1260.TA.26-13-j19-1.20.R |26 |1963|813 |279|72 |22 |6  |3  |1  |1  |1  |0  |1  |2  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |1  |0  |0  |0  |4  |\n",
-      "|8591071|928.TA.26-14-A-j19-1.10.R|9  |2598|1731|676|148|28 |10 |3  |4  |2  |1  |0  |1  |0  |0  |0  |0  |0  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
-      "|8591306|407.TA.26-17-j19-1.1.H   |14 |1501|688 |219|90 |43 |31 |14 |11 |4  |2  |2  |2  |3  |1  |0  |0  |0  |1  |0  |0  |0  |0  |1  |0  |0  |0  |0  |0  |0  |1  |1  |\n",
-      "|8590461|781.TA.26-485-j19-1.8.H  |14 |418 |275 |102|16 |2  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
-      "|8591065|370.TA.26-752-j19-1.5.R  |6  |307 |121 |24 |3  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
-      "|8591445|4117.TA.26-31-j19-1.26.R |1  |410 |161 |138|134|133|79 |52 |37 |33 |10 |7  |5  |2  |1  |1  |1  |1  |0  |2  |0  |0  |0  |0  |1  |0  |0  |0  |0  |0  |1  |1  |\n",
-      "+-------+-------------------------+---+----+----+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
+      "12309\n",
+      "+-------+-------------------------+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
+      "|stop_id|trip_id                  |-1 |0  |1  |2  |3  |4  |5  |6  |7  |8  |9  |10 |11 |12 |13 |14 |15 |16 |17 |18 |19 |20 |21 |22 |23 |24 |25 |26 |27 |28 |29 |30 |\n",
+      "+-------+-------------------------+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
+      "|8503003|286.TA.26-11-j19-1.80.H  |0  |395|75 |23 |5  |1  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503089|40.TA.26-4-B-j19-1.1.R   |0  |378|238|194|70 |46 |30 |11 |9  |8  |6  |4  |2  |2  |1  |0  |2  |2  |1  |0  |1  |1  |0  |0  |0  |0  |0  |1  |0  |0  |0  |0  |\n",
+      "|8503094|166.TA.26-4-B-j19-1.7.H  |0  |28 |158|102|50 |23 |21 |15 |7  |7  |1  |0  |1  |1  |2  |0  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503203|580.TA.26-8-A-j19-1.347.H|0  |659|504|246|110|58 |32 |13 |3  |1  |0  |5  |6  |3  |2  |2  |1  |1  |0  |2  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8502223|116.TA.26-14-j19-1.18.R  |0  |31 |144|56 |12 |3  |0  |0  |0  |0  |3  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503053|55.TA.79-10-B-j19-1.3.H  |0  |165|43 |7  |2  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503000|37.TA.26-15-j19-1.17.R   |8  |220|93 |47 |26 |9  |5  |5  |1  |1  |1  |0  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503006|297.TA.26-14-j19-1.41.H  |0  |123|197|59 |14 |11 |6  |1  |1  |0  |0  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503091|24.TA.26-4-B-j19-1.1.R   |0  |184|258|64 |7  |2  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "|8503147|150.TA.26-3-j19-1.12.H   |5  |470|119|43 |14 |7  |1  |3  |1  |0  |0  |1  |1  |0  |1  |0  |0  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
+      "+-------+-------------------------+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
       "only showing top 10 rows"
      ]
     }
    ],
    "source": [
-    "stop_times_distrib = spark.read.csv('data/lgpt_guys/distribution_1to1match.csv', \\\n",
+    "stop_times_distrib = spark.read.csv('data/lgpt_guys/distribution_geschaetzAndReal.csv', \\\n",
     "                                  header = True)\n",
     "print stop_times_distrib.count()\n",
     "stop_times_distrib.show(10, False)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "How many unique combination of stop_id / trip_id do we have ?"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 42,
+   "execution_count": 31,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "223579"
+      "12309"
      ]
     }
    ],
    "source": [
     "stop_times_distrib.select(\"stop_id\",\"trip_id\").distinct().count()"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "How many seems to have an empty line ? "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 54,
+   "execution_count": 32,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "8"
+      "4"
      ]
     }
    ],
    "source": [
     "stop_times_distrib.filter( (stop_times_distrib['-1'] == 0) &\\\n",
     "                           (stop_times_distrib['0'] == 0) &\\\n",
     "                           (stop_times_distrib['1'] == 0) &\\\n",
     "                           (stop_times_distrib['2'] == 0) ).count()"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "Actually there is no line with all values equal to zero : it would not have been assembled at the pivot stage. Now we want to see how many of the `stop_times_curated` lines we can get from this table :"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 77,
+   "execution_count": 33,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "+--------------------+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+\n",
-      "|                 key|  -1|   0|   1|   2|   3|   4|   5|   6|   7|   8|   9|  10|  11|  12|  13|  14|  15|  16|  17|  18|  19|  20|  21|  22|  23|  24|  25|  26|  27|  28|  29|  30|\n",
-      "+--------------------+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+\n",
-      "|1.TA.1-231-j19-1....|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|\n",
-      "|1.TA.1-231-j19-1....|   0|  78|  21|   6|   3|   1|   1|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
-      "|1.TA.1-231-j19-1....|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|\n",
-      "|1.TA.1-231-j19-1....|   0| 170|  42|   4|   2|   0|   2|   2|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
-      "|1.TA.1-231-j19-1....|   0| 170|  42|   4|   2|   0|   2|   2|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
-      "|1.TA.1-231-j19-1....|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|\n",
-      "|1.TA.1-231-j19-1....|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|\n",
-      "|1.TA.1-231-j19-1....|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|\n",
-      "|1.TA.1-231-j19-1....|   2|  94|   9|   2|   2|   1|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
-      "|1.TA.1-231-j19-1....|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|\n",
-      "|1.TA.1-231-j19-1....|   0|  78|  25|   6|   0|   0|   1|   1|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
-      "|1.TA.1-231-j19-1....|   4| 194|  16|   4|   0|   4|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
-      "|1.TA.1-231-j19-1....|   4| 194|  16|   4|   0|   4|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
-      "|1.TA.1-231-j19-1....|   6| 188|  18|   4|   2|   2|   2|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
-      "|1.TA.1-231-j19-1....|   6| 188|  18|   4|   2|   2|   2|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
-      "|1.TA.1-231-j19-1....|   5|  93|   8|   2|   1|   2|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
-      "|1.TA.1-231-j19-1....|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|\n",
-      "|1.TA.1-231-j19-1....|   0|  93|  12|   3|   1|   2|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|   0|\n",
-      "|1.TA.1-44-j19-1.1...|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|\n",
-      "|1.TA.1-44-j19-1.1...|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|null|\n",
-      "+--------------------+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+\n",
+      "+--------------------+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
+      "|                 key| -1|  0|  1|  2|  3|  4|  5|  6|  7|  8|  9| 10| 11| 12| 13| 14| 15| 16| 17| 18| 19| 20| 21| 22| 23| 24| 25| 26| 27| 28| 29| 30|\n",
+      "+--------------------+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
+      "|10.TA.1-11-B-j19-...|  0|  2|  2|  1|  3|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|10.TA.1-11-B-j19-...|  0|  3|  2|  2|  3|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|10.TA.1-11-B-j19-...|  0|  0|  4|  4|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|10.TA.1-11-B-j19-...|  0|  1|  5|  3|  1|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  1|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|10.TA.1-11-B-j19-...|  0|  1|  3|  4|  1|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|10.TA.26-912-j19-...|  0|  0|  1|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|10.TA.79-10-B-j19...|  1|129|143| 71| 33| 17| 11|  4|  3|  3|  1|  1|  1|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|10.TA.79-10-B-j19...|  0|333| 40| 22|  9|  5|  3|  4|  1|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|10.TA.79-10-B-j19...|  1|340| 37| 21|  6|  2|  6|  2|  2|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|10.TA.79-10-B-j19...|  0|  0|177| 23|  7|  0|  1|  0|  1|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|10.TA.79-10-B-j19...|  0|266| 81| 33| 15| 10|  3|  4|  3|  1|  0|  2|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|10.TA.79-10-B-j19...|  0|142|139| 69| 31| 16|  8|  5|  4|  1|  1|  1|  1|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|100.TA.26-6-A-j19...|  0|325| 62| 27|  1|  2|  2|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|100.TA.26-6-A-j19...|  0|310| 84| 15|  5|  4|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|100.TA.26-6-A-j19...|  0|330| 73|  9|  4|  2|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|100.TA.26-6-A-j19...|  0|257|103| 42| 14|  1|  1|  1|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|100.TA.26-6-A-j19...|  0| 69|228| 88| 23|  8|  1|  2|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|100.TA.26-6-A-j19...|  0|300| 86| 22|  8|  1|  0|  2|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|100.TA.26-6-A-j19...|  0|200|158| 38| 19|  2|  0|  1|  1|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "|100.TA.26-6-A-j19...|  0|349| 47| 14|  7|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|  0|\n",
+      "+--------------------+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
       "only showing top 20 rows"
      ]
     }
    ],
    "source": [
     "stop_times_final = stop_times_curated.join(stop_times_distrib,\\\n",
     "                                           on = ['stop_id', 'trip_id'],\\\n",
-    "                                           how = 'left_outer').drop('stop_id_raw')\\\n",
+    "                                           how = 'inner').drop('stop_id_raw')\\\n",
     ".orderBy('trip_id', 'stop_id')\\\n",
     ".withColumn('key2', concat(col('trip_id'), lit('__'), col('stop_id')))\\\n",
     ".drop('trip_id').drop('stop_id')\\\n",
     ".select(col('key2').alias('key'), \"*\")\\\n",
     ".drop('key2')\n",
     "\n",
     "stop_times_final.show(20)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "We still have null values. Let's count how many null we have on the full table"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 81,
+   "execution_count": 34,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "reference table stop_times number of lines : 247920\n",
-      "distribution table number of lines         : 247920\n",
-      "Number of missing keys in distribution     : 23503"
+      "reference table stop_times number of lines : 250777\n",
+      "distribution table number of lines         : 12309\n",
+      "Number of missing keys in distribution     : 0"
      ]
     }
    ],
    "source": [
     "print \"reference table stop_times number of lines : {}\".format(stop_times_curated.count())\n",
     "print \"distribution table number of lines         : {}\".format(stop_times_final.count())\n",
     "print \"Number of missing keys in distribution     : {}\".format(stop_times_final.filter(stop_times_final['0'].isNull()).count())"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "This represent around 10% of missing data. This is not so bad.\n",
-    "\n",
     "We write two version of the table : one with missing values, and one with missing values filled with '1',allowing development of next steps in the meantime (filling these values with a better approach is discussed in next section _Recovering missing distributions_)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 108,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     }
    ],
    "source": [
     "stop_times_final_fill1 = stop_times_final.na.fill(1) # not working, not IntegerType ..."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 110,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "+-----------------------------+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
       "|key                          |-1 |0  |1  |2  |3  |4  |5  |6  |7  |8  |9  |10 |11 |12 |13 |14 |15 |16 |17 |18 |19 |20 |21 |22 |23 |24 |25 |26 |27 |28 |29 |30 |\n",
       "+-----------------------------+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
       "|1.TA.1-231-j19-1.1.H__8502553|1  |1  |1  |1  |1  |1  |1  |1  |1  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
       "|1.TA.1-231-j19-1.1.H__8502879|0  |78 |21 |6  |3  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
       "|1.TA.1-231-j19-1.1.H__8502955|1  |1  |1  |1  |1  |1  |1  |1  |1  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
       "|1.TA.1-231-j19-1.1.H__8503598|0  |170|42 |4  |2  |0  |2  |2  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
       "|1.TA.1-231-j19-1.1.H__8503598|0  |170|42 |4  |2  |0  |2  |2  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
       "|1.TA.1-231-j19-1.1.H__8572600|1  |1  |1  |1  |1  |1  |1  |1  |1  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
       "|1.TA.1-231-j19-1.1.H__8572601|1  |1  |1  |1  |1  |1  |1  |1  |1  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
       "|1.TA.1-231-j19-1.1.H__8572602|1  |1  |1  |1  |1  |1  |1  |1  |1  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
       "|1.TA.1-231-j19-1.1.H__8572603|2  |94 |9  |2  |2  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
       "|1.TA.1-231-j19-1.1.H__8572747|1  |1  |1  |1  |1  |1  |1  |1  |1  |1  |1  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |0  |\n",
       "+-----------------------------+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n",
       "only showing top 10 rows"
      ]
     }
    ],
    "source": [
     "# This contains the list of columns where we apply replace() function\n",
     "all_column_names = stop_times_final.columns\n",
     "columns_to_remove = ['key']\n",
     "columns_for_replacement = [i for i in all_column_names if i not in columns_to_remove]\n",
     "\n",
     "# Doing the replacement on all the requisite columns\n",
     "for i in columns_for_replacement:\n",
     "    stop_times_final_fill1 = stop_times_final_fill1.withColumn(i,when((col(i).isNull()),int(int(i)<=10))\\\n",
     "                                                               .otherwise(col(i).cast(IntegerType())))\n",
     "stop_times_final_fill1.show(10, False)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 111,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     }
    ],
    "source": [
     "stop_times_final.write.csv('data/lgpt_guys/distribution_1to1match_wNull.csv', \\\n",
     "                                  header = True, mode=\"overwrite\")\n",
     "stop_times_final_fill1.write.csv('data/lgpt_guys/distribution_1to1match_fill1.csv', \\\n",
-    "                                  header = True, mode=\"overwrite\")\n"
+    "                                  header = True, mode=\"overwrite\")"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "### Use local python to make definitive table with right ordering \n",
     "\n",
     "We first use the tables where null values were filled with 1 and 0 "
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 30,
    "metadata": {},
+   "outputs": [],
+   "source": [
+    "username = 'acoudray'\n",
+    "stop_times_final_fill1.write.csv(\"/user/{}/distribution_1to1match_fill1.csv\".format(username), \\\n",
+    "                                 header = True, mode = 'overwrite')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
-    },
+    }
+   ],
+   "source": [
+    "# for geschaetz only\n",
+    "stop_times_final.write.csv(\"/user/{}/distribution_1to1match_geschaetz.csv\".format(username), \\\n",
+    "                                 header = True, mode = 'overwrite')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
     {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "An error was encountered:\n",
-      "name 'stop_times_final_fill1' is not defined\n",
-      "Traceback (most recent call last):\n",
-      "NameError: name 'stop_times_final_fill1' is not defined\n",
-      "\n"
-     ]
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
     }
    ],
    "source": [
-    "username = 'acoudray'\n",
-    "stop_times_final_fill1.write.csv(\"/user/{}/distribution_1to1match_fill1.csv\".format(username), \\\n",
+    "# for geschaetz and real only\n",
+    "stop_times_final.write.csv(\"/user/{}/distribution_1to1match_geschaetzAndReal.csv\".format(username), \\\n",
     "                                 header = True, mode = 'overwrite')"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 34,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
        "model_id": "",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
        "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "+--------------------+-------+------------+--------------+-------------+-----------+-------------+--------+-----------+----------+--------------------+------------+\n",
       "|             trip_id|stop_id|arrival_time|departure_time|stop_sequence|pickup_type|drop_off_type|stop_int|   route_id|sequence_1|              trip_1|   route_int|\n",
       "+--------------------+-------+------------+--------------+-------------+-----------+-------------+--------+-----------+----------+--------------------+------------+\n",
       "|1.TA.1-231-j19-1.1.H|8572747|    09:37:00|      09:37:00|            1|          0|            0|     500|1-231-j19-1|      10.0|1.TA.1-231-j19-1.1.H|592705486850|\n",
       "|1.TA.1-231-j19-1.1.H|8573721|    09:50:00|      09:50:00|           10|          0|            0|     599|1-231-j19-1|      11.0|1.TA.1-231-j19-1.1.H|592705486850|\n",
       "|1.TA.1-231-j19-1.1.H|8503598|    09:53:00|      09:53:00|           11|          0|            0|     401|1-231-j19-1|      12.0|1.TA.1-231-j19-1.1.H|592705486850|\n",
       "+--------------------+-------+------------+--------------+-------------+-----------+-------------+--------+-----------+----------+--------------------+------------+\n",
       "only showing top 3 rows"
      ]
     }
    ],
    "source": [
     "username = 'acoudray'\n",
     "stop_times_curated.write.csv(\"/user/{}/stop_times_curated_sbbCompatible\".format(username), \\\n",
     "                                 header = True, mode = 'overwrite')\n",
     "stop_times_curated.show(3)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 36,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[('10.TA.1-11-B-j19-1.1.R__8590314',\n",
+       "  array([0, 2, 2, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0])),\n",
+       " ('10.TA.1-11-B-j19-1.1.R__8590317',\n",
+       "  array([0, 3, 2, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0])),\n",
+       " ('10.TA.1-11-B-j19-1.1.R__8594304',\n",
+       "  array([0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0])),\n",
+       " ('10.TA.1-11-B-j19-1.1.R__8594307',\n",
+       "  array([0, 1, 5, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,\n",
+       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0])),\n",
+       " ('10.TA.1-11-B-j19-1.1.R__8594310',\n",
+       "  array([0, 1, 3, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0])),\n",
+       " ('10.TA.26-912-j19-1.2.R__8576195',\n",
+       "  array([0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0])),\n",
+       " ('10.TA.79-10-B-j19-1.2.R__8503051',\n",
+       "  array([  1, 129, 143,  71,  33,  17,  11,   4,   3,   3,   1,   1,   1,\n",
+       "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
+       "           0,   0,   0,   0,   0,   0])),\n",
+       " ('10.TA.79-10-B-j19-1.2.R__8503052',\n",
+       "  array([  0, 333,  40,  22,   9,   5,   3,   4,   1,   0,   0,   0,   0,\n",
+       "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
+       "           0,   0,   0,   0,   0,   0])),\n",
+       " ('10.TA.79-10-B-j19-1.2.R__8503053',\n",
+       "  array([  1, 340,  37,  21,   6,   2,   6,   2,   2,   0,   0,   0,   0,\n",
+       "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
+       "           0,   0,   0,   0,   0,   0])),\n",
+       " ('10.TA.79-10-B-j19-1.2.R__8503054',\n",
+       "  array([  0,   0, 177,  23,   7,   0,   1,   0,   1,   0,   0,   0,   0,\n",
+       "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
+       "           0,   0,   0,   0,   0,   0]))]"
+      ]
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "%local\n",
     "\n",
     "from hdfs3 import HDFileSystem\n",
     "import pandas as pd\n",
     "import numpy as np \n",
     "import pickle \n",
     "import gzip\n",
     "from itertools import islice\n",
     "\n",
     "hdfs = HDFileSystem(host='hdfs://iccluster044.iccluster.epfl.ch', port=8020, user='ebouille')\n",
     "\n",
     "username = 'acoudray'\n",
     "\n",
     "# Load distribution file from HDFS and concatenate individual csv\n",
-    "distrib_files = hdfs.glob('/user/{}/distribution_1to1match_fill1.csv/*.csv'.format(username))\n",
+    "distrib_files = hdfs.glob('/user/{}/distribution_1to1match_geschaetzAndReal.csv/*.csv'.format(username))\n",
     "distrib = pd.DataFrame()\n",
     "for file in distrib_files:\n",
     "    with hdfs.open(file) as f:\n",
     "        distrib = distrib.append(pd.read_csv(f))\n",
     "distrib = distrib.set_index('key')\n",
     "\n",
     "# zip index and values to get {key : np.array()} shape \n",
     "d = dict(zip(distrib.index, np.array(distrib.values)))\n",
     "\n",
     "# Write it to local \n",
-    "with gzip.open(\"../data/distributions.pickle\", \"wb\") as output_file:\n",
+    "with gzip.open(\"../data/distributions_geschaetzAndReal.pkl.gz\", \"wb\") as output_file:\n",
     "    pickle.dump(d, output_file)\n",
     "\n",
     "# Functon to take a slice from a dictionnary - head equivalent\n",
     "def take(n, iterable):\n",
     "    \"Return first n items of the iterable as a list\"\n",
     "    return list(islice(iterable, n))\n",
     "\n",
     "# display a slice of it\n",
-    "take(50, d.items())"
+    "take(10, d.items())"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "How many RAM does the dictionnary occupy when it is open ? Open pickle and calculate amount of memory occupied using _resource_ lib"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 41,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "length of dict :  246968\n",
       "Data size is: 106968218\n"
      ]
     }
    ],
    "source": [
     "%local \n",
     "\n",
     "import pickle \n",
     "import gzip\n",
     "import sys\n",
     "import os\n",
     "import resource\n",
     "\n",
     "with gzip.open(\"../data/distributions.pickle\", \"rb\") as input_file:\n",
     "    d = pickle.load(input_file)\n",
     "    \n",
     "\n",
     "d['1290.TA.26-32-j19-1.12.H__8591151']\n",
     "print('length of dict : ',len(d))\n",
     "\n",
     "def getsizeof_r(obj):\n",
     "    total = 0\n",
     "    if isinstance(obj, list):\n",
     "        for i in obj:\n",
     "            total += getsizeof_r(i)\n",
     "    elif isinstance(obj, dict):\n",
     "        for k, v in obj.items():\n",
     "            total += getsizeof_r(k) + getsizeof_r(v)\n",
     "    else:\n",
     "        total += sys.getsizeof(obj)\n",
     "    return total\n",
     "\n",
     "print('Data size is: {}'.format(getsizeof_r(d)))"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "How many time does it take to access elements in the dictionnary ?"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "[   0 1008  405  207   95   39   25   11    5    3    0    0    0    0\n",
       "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
       "    0    0    0    0]\n",
       "running time to get value from key when exists : 0.0004305839538574219\n",
       "\n",
       "KEY ERROR: .26-32-j19-1.12.H__8591151 not found un distribution dictionnary\n",
       "running time to get error when key does NOT exists : 0.00010466575622558594\n",
       "\n"
      ]
     }
    ],
    "source": [
     "%local\n",
     "\n",
     "import pickle \n",
     "import gzip\n",
     "import time\n",
     "\n",
     "def get_distribution(key, dico):\n",
     "    if key in dico:\n",
     "        print(dico[key])\n",
     "    else:\n",
     "        print(\"KEY ERROR: {} not found un distribution dictionnary\".format(key))\n",
     "        \n",
     "with gzip.open(\"../data/distributions.pickle\", \"rb\") as input_file:\n",
     "    d = pickle.load(input_file)\n",
     "    \n",
     "this_key = '1290.TA.26-32-j19-1.12.H__8591151'\n",
     "\n",
     "start = time.time()\n",
     "get_distribution(this_key, d)\n",
     "end = time.time()\n",
     "print(\"running time to get value from key when exists : {}\\n\".format(end - start))\n",
     "\n",
     "start = time.time()\n",
     "get_distribution(this_key.replace('1290.TA',''), d)\n",
     "end = time.time()\n",
     "print(\"running time to get error when key does NOT exists : {}\\n\".format(end - start))"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "when key exists we access it in $5\\cdot10^{-4}$ seconds and when it does not exists error message is displayed in $1\\cdot10^{-4}$ seconds. Should be more than enough to be called multiple time when using raptor."
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "### Make function to compute probabilities from distributions\n",
     "\n",
     "We make a script that takes a key, a arrival time and a departure time to compute a probability to be at least 2 minutes ahead for transfert. We assume that with less than 2 minutes, we miss the transfert.\n",
     "\n",
     "We will use a Poisson distribution to compute this probability using its cumulative distribution.\n",
     "\n",
     "A Poisson Process meets the following criteria (in reality many phenomena modeled as Poisson processes don’t meet these exactly):\n",
     "- Events are independent of each other. The occurrence of one event does not affect the probability another event will occur.\n",
     "- The average rate (events per time period) is constant.\n",
     "- Two events cannot occur at the same time.\n",
     "\n",
     "Bounds for the tail probabilities of a Poisson random variable ${\\displaystyle X\\sim \\operatorname {Pois} (\\lambda )}$ can be derived using a Chernoff bound argument: \n",
     "\n",
     "$${\\displaystyle P(X\\leq x)\\leq {\\frac {(e\\lambda )^{x}e^{-\\lambda }}{x^{x}}},{\\text{ for }}x<\\lambda .}$$\n",
     "\n",
     "So in our case all we need to find is $\\lambda$. The positive real number λ is equal to the expected value of X and also to its variance :\n",
     "\n",
     "$${\\lambda =\\operatorname {E} (X)=\\operatorname {Var} (X)}$$\n",
     "\n",
     "We can easily find $\\lambda$ by finding the _average number of success per unit time_. We have a distribution going from -1 to +30, therefore we will iterate over it and sum up all successes $x_t$ at each time point $t$, for all our time points. \n",
     "\n",
     "$$ {\\lambda = \\frac{1}{N} \\displaystyle\\sum_{t=-1}^{N=30} x_t \\cdot t}$$"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 40,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "lambda (expectation given distribution):  4.0 \n",
       "\n",
       "Probability of success for transfer time = 5.0 minutes :  0.7851303870304052\n"
      ]
     },
     {
      "data": {
       "text/plain": [
        "0.7851303870304052"
       ]
      },
      "execution_count": 40,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "%local\n",
     "\n",
     "import pickle \n",
     "import gzip\n",
     "import time\n",
     "import math \n",
     "import datetime\n",
     "import time\n",
     "from scipy.stats import poisson\n",
     "\n",
     "with gzip.open(\"../data/distributions.pickle\", \"rb\") as input_file:\n",
     "    d = pickle.load(input_file)\n",
     "    \n",
     "def get_distrib(key, dico):\n",
     "    if key in dico:\n",
     "        return dico[key]\n",
     "    else:\n",
     "        raise ValueError(\"KEY ERROR: {} not found un distribution dictionnary\".format(key))\n",
     "    \n",
     "def evaluate_lamda(distrib):\n",
     "    # First calculate total number of measures N\n",
     "    N = 0\n",
     "    for x in distrib:\n",
     "        N += x\n",
     "\n",
     "    lambda_p = 0 # expectation - we want to calculate it\n",
     "    t        = -1 # time = index - 1\n",
     "\n",
     "    for x in distrib:\n",
     "        lambda_p += t*x\n",
     "        t += 1\n",
     "\n",
     "    # calculate lambda which is the expectation of x\n",
     "    if N > 0:\n",
     "        lambda_p /= N \n",
     "        print('lambda (expectation given distribution): ',lambda_p, '\\n')\n",
     "        return lambda_p\n",
     "    else : \n",
     "        raise ValueError(\"ERROR : {} distribution has 0 counts\".format(key))\n",
     "        #print('Returning 1 to avoid later problem... \\n')\n",
     "        return 1\n",
     "\n",
     "def process_time(str_time):\n",
     "    x = time.strptime(str_time,'%H:%M')\n",
     "    return datetime.timedelta(hours=x.tm_hour,minutes=x.tm_min,seconds=x.tm_sec).total_seconds()\n",
     "\n",
     "def get_transfer_time(arr_time, dep_time, delta=2.0):\n",
     "    diff_time_min = ( process_time(dep_time) - process_time(arr_time) ) / 60\n",
     "    return diff_time_min - delta\n",
     "\n",
     "def poisson_proba(trip_id, stop_id, arr_time, dep_time, dico):\n",
     "    # Generate key from trip_id / stop_id \n",
     "    key = str(trip_id) + '__' + str(stop_id[0:7]) # 7 first char to be sbb-compatible\n",
     "\n",
     "    # Get distribution from dictionnary\n",
     "    distrib = get_distrib(key, dico)\n",
     "    \n",
     "    # Calculate transfer time at disposal \n",
     "    T = get_transfer_time(arr_time, dep_time)\n",
     "    \n",
     "    # Get lambda value to calculate proba\n",
     "    lambda_p = evaluate_lamda(distrib)\n",
     "\n",
     "    # Get proba\n",
     "    poisson_p = poisson.cdf(T, lambda_p)\n",
     "    print('Probability of success for transfer time = {} minutes : '.format(T),poisson_p)\n",
     "\n",
     "    return poisson_p\n",
     "\n",
     "# 129.TA.90-173-Y-j19-1.1.H__8530643\n",
     "# input data :\n",
     "trip_id = '129.TA.90-173-Y-j19-1.1.H'\n",
     "stop_id = '8530643'\n",
     "arrival_time = '07:45'\n",
     "departure_time = '07:52'\n",
     "Pr = poisson_proba(trip_id, stop_id, arrival_time, departure_time, d)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "# trip_id missing \n",
     "\n",
     "stop_id , time , transport_type -> estimate proba \n",
     "\n",
     "# Make recovery tables\n",
     "1500 x 24 x 5 = 180'000\n",
     "\n",
     "# Validate recovery table \n"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
    "display_name": "PySpark",
    "language": "",
    "name": "pysparkkernel"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "python",
     "version": 3
    },
    "mimetype": "text/x-python",
    "name": "pyspark",
    "pygments_lexer": "python3"
   }
  },
  "nbformat": 4,
  "nbformat_minor": 4
 }