Page MenuHomec4science
Files F110874222

eval_residual_summary_logs.py
No OneTemporary
Actions

File Metadata

Created
Mon, Apr 28, 13:49

eval_residual_summary_logs.py
View Options

import pandas as pd
import os
import xarray as xr
import pickle
import sys
import util
path = sys.argv[1]
dataset = sys.argv[2]
ELM = sys.argv[3]
models = sys.argv[4].split(',')
file_link = { 'time' : 'log_time.csv' ,
'error' : 'log_pred_mse.csv' ,
}
dirpath = os.path.join( path, 'datasets', dataset, ELM )
subdirs = os.listdir( dirpath )
# decalre merged log
merged_log = {}
log_list = []
# load all files:
for file_label in file_link:
for model in models:
try:
logfile_name = model + '_' + file_link[file_label]
logfile_dir = os.path.join( dirpath, logfile_name)
# load file and drop the cputime (not relevant for ELM as equal to cputime)
current_log = pd.read_csv( logfile_dir, index_col = 0)
# go through the list of collected variables again and rename them accoding to the model
for column_label in current_log.columns:
label, n_nodes = model.split('_')
ensemble_size = label.split('s')[1]
label_for_dict_entry = column_label
entry = current_log[column_label].to_frame().rename({ column_label : n_nodes }, axis = 1)
meta_information = pd.DataFrame( data = [ int(ensemble_size) ], index = ['n_ens'], columns = [n_nodes] )
entry = entry.append(meta_information)
# merge with the merged_log ( and set if the variable does not yet exist)
if label_for_dict_entry in merged_log:
merged_log[ label_for_dict_entry ] = merged_log[ label_for_dict_entry ].merge(entry,
left_index = True,
right_index = True,
how = 'outer')
else:
merged_log[ label_for_dict_entry ] = entry
except Exception as e: print(e)
for entry in merged_log:
# transpose the dataframes, then change the index to integer type and sort
merged_log[entry] = merged_log[entry].T
merged_log[entry].index = merged_log[entry].index.astype(int)
merged_log[entry] = merged_log[entry].sort_index()
logfile_out = util.find_name(os.path.join( dirpath, 'summary_merged_log.pickle' ))
with open(logfile_out, 'wb') as handle:
pickle.dump(merged_log, handle, protocol=pickle.HIGHEST_PROTOCOL)
print('Created file %s' %logfile_out)

Event Timeline

c4science · Help