Homec4science
Diffusion solar_potential (aw_solar) Browse

solarPOT/Solar/RF_importance_all.py
dd1c0248d00daw_solar

Last Change
Raw File

RF_importance_all.py
View Options

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import LinearSegmentedColormap
import pandas as pd
import xarray as xr
import os
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error as mse
from meteo_data import Meteo_Reader
from sklearn.externals import joblib
import util
import csv
data_path = os.path.abspath("/mnt/sda1/hyenergy/data/meteo")
# data_path = os.path.abspath("/Users/alinawalch/Documents/EPFL/data/meteo")
#yr = Meteo_Reader(os.path.join(data_path, 'raw_data'), ['SIS'])
#yr.read_yearly(2001)
#data = yr.make_table()
#cols = data.drop(columns = ['lon','lat','date','year','CH','day','timestamp','SIS']).columns
ttt = util.Timer()
print('Reading data from pickle file ...')
data = pd.read_pickle(os.path.join(data_path, 'datasets', 'hourly_all.pkl'))
ttt.stop()
cols = data.drop(columns = ['lon','lat','date','year','CH','day','timestamp','SIS']).columns
# Random Forest
n_est = 500
with open('inds_all.csv', 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(cols)
with open('stds_all.csv', 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(cols)
with open('importances_all.csv', 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(cols)
with open('RF_tests_idx.csv', 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(['month', 'hour'])
#################### MONTHLY ANALYSIS ##################################
for m in range(1,13):
for h in range(3,20):
i = (m-1)*17+(h-3)
print('\n\n Month = %d, hour = %d' %(m,h))
sample = data.loc[(data.hour==h) & (data.month==m)]
ftr_tbl = sample.drop(columns = ['lon','lat','date','year','CH','day','timestamp','SIS'])
tgt_tbl = sample[['SIS']]
print(ftr_tbl.columns)
ftrs = util.standardize(ftr_tbl, ftr_tbl.mean(), ftr_tbl.std())
tgts = util.standardize(tgt_tbl, tgt_tbl.mean(), tgt_tbl.std())
x0 = ftrs.as_matrix()
t0 = tgts.as_matrix().reshape((-1,))
x = ftrs.as_matrix()
t = tgts.as_matrix().reshape((-1,))
print(x.shape)
print(t.shape)
rf = RandomForestRegressor(n_estimators = n_est, max_depth = 100, n_jobs = -1)
print('Training random forest... ')
timer_0 = util.Timer()
rf.fit(x, t)
timer_0.stop()
importance = np.mean([tree.feature_importances_ for tree in rf.estimators_],axis=0)
std = np.std([tree.feature_importances_ for tree in rf.estimators_],axis=0)
ind = np.argsort(importance)[-30:]
print(importance)
print(std)
print(ind)
if np.sum(importance)> 0.0001:
with open('inds_all.csv', 'a') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(ind)
with open('stds_all.csv', 'a') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(std)
with open('importances_all.csv', 'a') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(importance)
with open('RF_tests_idx.csv', 'a') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow([m, h])
# plt.figure(figsize=(15,15))
# plt.title("Feature Importances for Random Forest (%d trees)" %n_est)
# plt.barh(range(30),importance[ind],
# color="r", xerr=std[ind], alpha=0.4, align="center")
# plt.yticks(range(30), feature_names[ind])
# plt.ylim([-1, len(ind)])
# plt.savefig("importances.png")
c4science · Help