"/home/walch/miniconda3/envs/py3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
" from ._conv import register_converters as _register_converters\n"
]
}
],
"source": [
"import os\n",
"import pandas as pd\n",
"import xarray as xr\n",
"import numpy as np\n",
"\n",
"import time\n",
"import util\n",
"\n",
"import matplotlib\n",
"%matplotlib notebook\n",
"import matplotlib.pyplot as plt"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Aims:\n",
"- Combine parallelised .nc files\n",
"- Remove outliers from the dataset"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"path = # PATH TO INTERMEDIATE FILES (output of poa_irradiance.py)\n",
+ "## Geographic transformation to attach x and y values to lon and lat"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "list_of_locations = G_all_nonzero.isel(timestamp = 0, day = 0).to_dataframe().reset_index()[['lat','lon']]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>lat</th>\n",
+ " <th>lon</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>45.750</td>\n",
+ " <td>5.750000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>45.750</td>\n",
+ " <td>5.770833</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>45.750</td>\n",
+ " <td>5.791667</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>45.750</td>\n",
+ " <td>5.812500</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>45.750</td>\n",
+ " <td>5.833333</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>...</th>\n",
+ " <td>...</td>\n",
+ " <td>...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>24818</th>\n",
+ " <td>47.875</td>\n",
+ " <td>10.666667</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>24819</th>\n",
+ " <td>47.875</td>\n",
+ " <td>10.687500</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>24820</th>\n",
+ " <td>47.875</td>\n",
+ " <td>10.708334</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>24821</th>\n",
+ " <td>47.875</td>\n",
+ " <td>10.729167</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>24822</th>\n",
+ " <td>47.875</td>\n",
+ " <td>10.750000</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "<p>24823 rows × 2 columns</p>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " lat lon\n",
+ "0 45.750 5.750000\n",
+ "1 45.750 5.770833\n",
+ "2 45.750 5.791667\n",
+ "3 45.750 5.812500\n",
+ "4 45.750 5.833333\n",
+ "... ... ...\n",
+ "24818 47.875 10.666667\n",
+ "24819 47.875 10.687500\n",
+ "24820 47.875 10.708334\n",
+ "24821 47.875 10.729167\n",
+ "24822 47.875 10.750000\n",
+ "\n",
+ "[24823 rows x 2 columns]"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "list_of_locations"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/alinawalch/miniconda3/envs/py3/lib/python3.7/site-packages/pyproj/crs/crs.py:53: FutureWarning: '+init=<authority>:<code>' syntax is deprecated. '<authority>:<code>' is the preferred initialization method. When making the change, be mindful of axis order changes: https://pyproj4.github.io/pyproj/stable/gotchas.html#axis-order-changes-in-proj-6\n",
+ "/Users/alinawalch/miniconda3/envs/py3/lib/python3.7/site-packages/pyproj/crs/crs.py:294: FutureWarning: '+init=<authority>:<code>' syntax is deprecated. '<authority>:<code>' is the preferred initialization method. When making the change, be mindful of axis order changes: https://pyproj4.github.io/pyproj/stable/gotchas.html#axis-order-changes-in-proj-6\n",
+ "/Users/alinawalch/miniconda3/envs/py3/lib/python3.7/site-packages/ipykernel_launcher.py:4: DeprecationWarning: This function is deprecated. See: https://pyproj4.github.io/pyproj/stable/gotchas.html#upgrading-to-pyproj-2-from-pyproj-1\n",
+ " after removing the cwd from sys.path.\n"
+ ]
+ }
+ ],
+ "source": [
+ "# transform x and y into lon and lat and append to list_of_locations\n",
+ " time (day, timestamp) datetime64[ns] dask.array<chunksize=(31, 154), meta=np.ndarray>"
+ ]
+ },
+ "execution_count": 28,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "communes_output"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/alinawalch/miniconda3/envs/py3/lib/python3.7/site-packages/dask/core.py:121: RuntimeWarning: invalid value encountered in greater\n",
+ " return func(*(_execute_task(a, cache) for a in args))\n",
+ "/Users/alinawalch/miniconda3/envs/py3/lib/python3.7/site-packages/dask/array/numpy_compat.py:40: RuntimeWarning: invalid value encountered in true_divide\n",
+ " x = np.divide(x1, x2, out)\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Computed hourly commune potential and saved output in 115.82s\n"
- "## Geographic transformation to attach x and y values to lon and lat"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {},
- "outputs": [],
- "source": [
- "list_of_locations = G_all_nonzero.isel(timestamp = 0, day = 0).to_dataframe().reset_index()[['lat','lon']]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 14,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "<div>\n",
- "<style scoped>\n",
- " .dataframe tbody tr th:only-of-type {\n",
- " vertical-align: middle;\n",
- " }\n",
- "\n",
- " .dataframe tbody tr th {\n",
- " vertical-align: top;\n",
- " }\n",
- "\n",
- " .dataframe thead th {\n",
- " text-align: right;\n",
- " }\n",
- "</style>\n",
- "<table border=\"1\" class=\"dataframe\">\n",
- " <thead>\n",
- " <tr style=\"text-align: right;\">\n",
- " <th></th>\n",
- " <th>lat</th>\n",
- " <th>lon</th>\n",
- " </tr>\n",
- " </thead>\n",
- " <tbody>\n",
- " <tr>\n",
- " <th>0</th>\n",
- " <td>45.750</td>\n",
- " <td>5.750000</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>1</th>\n",
- " <td>45.750</td>\n",
- " <td>5.770833</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>2</th>\n",
- " <td>45.750</td>\n",
- " <td>5.791667</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>3</th>\n",
- " <td>45.750</td>\n",
- " <td>5.812500</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>4</th>\n",
- " <td>45.750</td>\n",
- " <td>5.833333</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>...</th>\n",
- " <td>...</td>\n",
- " <td>...</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>24818</th>\n",
- " <td>47.875</td>\n",
- " <td>10.666667</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>24819</th>\n",
- " <td>47.875</td>\n",
- " <td>10.687500</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>24820</th>\n",
- " <td>47.875</td>\n",
- " <td>10.708334</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>24821</th>\n",
- " <td>47.875</td>\n",
- " <td>10.729167</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>24822</th>\n",
- " <td>47.875</td>\n",
- " <td>10.750000</td>\n",
- " </tr>\n",
- " </tbody>\n",
- "</table>\n",
- "<p>24823 rows × 2 columns</p>\n",
- "</div>"
- ],
- "text/plain": [
- " lat lon\n",
- "0 45.750 5.750000\n",
- "1 45.750 5.770833\n",
- "2 45.750 5.791667\n",
- "3 45.750 5.812500\n",
- "4 45.750 5.833333\n",
- "... ... ...\n",
- "24818 47.875 10.666667\n",
- "24819 47.875 10.687500\n",
- "24820 47.875 10.708334\n",
- "24821 47.875 10.729167\n",
- "24822 47.875 10.750000\n",
- "\n",
- "[24823 rows x 2 columns]"
- ]
- },
- "execution_count": 14,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "list_of_locations"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 15,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/Users/alinawalch/miniconda3/envs/py3/lib/python3.7/site-packages/pyproj/crs/crs.py:53: FutureWarning: '+init=<authority>:<code>' syntax is deprecated. '<authority>:<code>' is the preferred initialization method. When making the change, be mindful of axis order changes: https://pyproj4.github.io/pyproj/stable/gotchas.html#axis-order-changes-in-proj-6\n",
- "/Users/alinawalch/miniconda3/envs/py3/lib/python3.7/site-packages/pyproj/crs/crs.py:294: FutureWarning: '+init=<authority>:<code>' syntax is deprecated. '<authority>:<code>' is the preferred initialization method. When making the change, be mindful of axis order changes: https://pyproj4.github.io/pyproj/stable/gotchas.html#axis-order-changes-in-proj-6\n",
- "/Users/alinawalch/miniconda3/envs/py3/lib/python3.7/site-packages/ipykernel_launcher.py:4: DeprecationWarning: This function is deprecated. See: https://pyproj4.github.io/pyproj/stable/gotchas.html#upgrading-to-pyproj-2-from-pyproj-1\n",
- " after removing the cwd from sys.path.\n"
- ]
- }
- ],
- "source": [
- "# transform x and y into lon and lat and append to list_of_locations\n",
"Meteo data loaded from /Volumes/Seagate Backup Plus Drive/DOKUMENTE/EPFL/data/Meteo_data/Meteoswiss_satellite/raw_data/2010_physical_potential_interp.nc\n",
"Rooftop data loaded from /Volumes/Seagate Backup Plus Drive/DOKUMENTE/EPFL/data/Results/PV_w_uncertainty/CH_ROOFS_all_replaced.csv\n",
"Data will be saved to hourly_poa.nc\n",
"Loading slice: 100 rows, starting from ID 1\n"
]
}
],
"source": [
"fp = '/Volumes/Seagate Backup Plus Drive/DOKUMENTE/EPFL/data/'\n",
"/Users/alinawalch/miniconda3/envs/py3/lib/python3.7/site-packages/pyproj/crs/crs.py:53: FutureWarning: '+init=<authority>:<code>' syntax is deprecated. '<authority>:<code>' is the preferred initialization method. When making the change, be mindful of axis order changes: https://pyproj4.github.io/pyproj/stable/gotchas.html#axis-order-changes-in-proj-6\n",
"/Users/alinawalch/miniconda3/envs/py3/lib/python3.7/site-packages/pyproj/crs/crs.py:294: FutureWarning: '+init=<authority>:<code>' syntax is deprecated. '<authority>:<code>' is the preferred initialization method. When making the change, be mindful of axis order changes: https://pyproj4.github.io/pyproj/stable/gotchas.html#axis-order-changes-in-proj-6\n",
"/Users/alinawalch/miniconda3/envs/py3/lib/python3.7/site-packages/ipykernel_launcher.py:4: DeprecationWarning: This function is deprecated. See: https://pyproj4.github.io/pyproj/stable/gotchas.html#upgrading-to-pyproj-2-from-pyproj-1\n",
"print('Number of buildings (with building period information) in Geneva canton: %d' %len(buildings_GVA_with_baup))\n",
"print('Percentage of buildings (with building period information) in Geneva canton: %.2f%%' %(len(buildings_GVA_with_baup)/len(buildings_with_baup)*100))"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Percentage of buildings in GVA canton built before 2000: 92.72%\n",
"Percentage of buildings in GVA canton built after 2000: 7.28%\n"