Homec4science
Diffusion Machine Learning ECMO (master) Browse

ML_ECMO/process_data.py
95b55da20f4bmaster

Last Change
Raw File

process_data.py
View Options

import csv
import numpy as np
import pandas as pd
from openpyxl import load_workbook
from param_str import param_str, findL, replaceL, inputs
def change_time_format(path, n):
for i in range(n):
print("Changing time format in the file " + str(i) + "...")
wb = load_workbook(path + '/Signals10_' + str(i) + '.xlsx')
ws = wb.active
ws.delete_rows(1, 8)
for row in ws.iter_rows(min_row=1, min_col=5, max_col=5):
for cell in row:
cell.number_format = 'General'
wb.save(path + '/Signals_' + str(n) + '_.xlsx')
def create_patient_file_and_extract_time_param(path, n):
i = 0
print("\033[1;32;48m Creating file for patient " + str(i) + " and extracting range of time and parameter...")
csvfile = open(path + '/Signals_' + str(n) + '_.csv')
output = open(path + '/data_tmp/Signals_' + str(i) + '.csv', mode='w')
reader = csv.reader(csvfile, delimiter=';')
first_row = next(reader)[0:5]
patient, param_name, time_val = first_row[0], first_row[1], first_row[-1]
times, time_patient = [], [time_val.replace(',', '.')]
params, file_params = [], [param_name]
for row in reader:
if row[1] != param_name and row[0] == patient:
param_name = row[1]
file_params.append(param_name)
if row[-1] != time_val and row[0] == patient:
time_val = row[-1]
if time_val.replace(',', '.') not in time_patient:
time_patient.append(time_val.replace(',', '.'))
# If the patient change...
if row[0] != patient:
# ... we get the new patient...
patient = row[0]
# ... close the file of the previous patient...
output.close()
# ... and open a new one for this one.
output = open(path + '/data_tmp/Signals_' + str(i) + '.csv', mode='w')
writer = csv.writer(output, delimiter=';')
time_patient.sort()
times.append(time_patient)
params.append(file_params)
time_patient = [row[-1].replace(',', '.')]
file_params = [row[1]]
time_val, param_name = row[-1].replace(',', '.'), row[1]
i = i + 1
print("\033[1;32;48m Creating file for patient " + str(i) + " and extracting range of time and parameter...")
writer.writerow(row)
# Close original file
csvfile.close()
output.close()
return times, params
# Create a single file for each patient
def create_file_for_each_patient(path, n):
i = 0
print("\033[1;32;48m Creating files for each patient ...")
csvfile = open(path + '/Signals_' + str(n) + '_.csv')
output = open(path + '/data_tmp/Signals_' + str(i) + '.csv', mode='w')
reader = csv.reader(csvfile, delimiter=';')
writer = csv.writer(output, delimiter=';')
first_row = next(reader)[0:5]
patient, time_val = first_row[0], first_row[-1]
for row in reader:
# If the patient change...
if row[0] != patient:
# ... we get the new patient...
patient = row[0]
# ... close the file of the previous patient...
output.close()
# ... and open a new one for this one.
output = open(path + '/data_tmp/Signals_' + str(i) + '.csv', mode='w')
writer = csv.writer(output, delimiter=';')
i = i + 1
writer.writerow(row)
# Close original file
csvfile.close()
output.close()
# Create a single file for each patient
def create_file_for_each_patient_mult_files(path, n):
i = 0
for j in range(n):
print("\033[1;32;48m Creating files for each patient ...")
csvfile = open(path + '/raw_data/Signals10_' + str(j) + '.csv')
output = open(path + '/data_tmp/Signals_' + str(i) + '.csv', mode='w')
reader = csv.reader(csvfile, delimiter=';')
writer = csv.writer(output, delimiter=';')
first_row = next(reader)[0:5]
patient, time_val = first_row[0], first_row[-1]
for row in reader:
# If the patient change...
if row[0] != patient:
# ... we get the new patient...
patient = row[0]
# ... close the file of the previous patient...
output.close()
# ... and open a new one for this one.
output = open(path + '/data_tmp/Signals_' + str(i) + '.csv', mode='w')
writer = csv.writer(output, delimiter=';')
i = i + 1
writer.writerow(row)
# Close original file
csvfile.close()
output.close()
def get_time_param_range(path, i, times, params):
print("\033[1;34;48m Getting time range and parameters range for file ", str(i), "...")
# Open the files
csvfile = open(path + '/data_tmp/Signals_' + str(i) + '.csv', mode='r')
output = open(path + '/data/Signals_' + str(i) + '.csv', mode='w')
# Initialize the reader
reader = csv.reader(csvfile, delimiter=';')
# Get the time and the parameter name of the first row
first_row = next(reader)
param_name, time_val = first_row[1], first_row[-1]
# Replace the comma of time by a point in order to convert it later in a float
time = [time_val.replace(',', '.')]
file_params = [param_name]
for row in reader:
if row[1] != param_name:
param_name = row[1]
file_params.append(param_name)
if row[-1] != time_val:
time_val = row[-1]
if time_val.replace(',', '.') not in time:
time.append(time_val.replace(',', '.'))
time.sort()
times.append(time)
params.append(file_params)
csvfile.close()
output.close()
def get_time_param_range_all(path, n):
times, params = [], []
# Get time range et parameters range for each patient
for i in range(n):
get_time_param_range(path, i, times, params)
return times, params
def rearrange_file(path, times, max_params, params, param_str, i):
print("\033[1;35;48m Rearranging file ", str(i), '...')
# Open files
csvfile = open(path + '/data_tmp/Signals_' + str(i) + '.csv')
output = open(path + '/data/Signals_' + str(i) + '.csv', mode='w')
writer = csv.writer(output, delimiter=';')
reader = csv.reader(csvfile, delimiter=';')
# Write header
#header = params[i]
writer.writerow(max_params)
time_min = min(times[i])
#file_params = params[i]
# Create a tab length num_params * num_time_steps. We will write in the file at the end
tab = [[np.nan] * len(max_params) for _ in range(len(times[i]))]
time_arr = np.array(times[i])
time_arr = time_arr.astype(np.float)
# Convert list into np.array in order to make it easier to assign its values
tab = np.array(tab)
# Re-scaling the time range
tab[:, 0] = time_arr - float(time_min)
# For each line...
for row in reader:
# ... we take the parameter and the time of the current and look at its index in the time table and
# in the parameter table
time, param = row[-1].replace(',', '.'), row[1]
correct_param_name(param)
idx, idy = times[i].index(time), max_params.index(param)
param_value = row[2]
# If the parameter does not have a string value...
if row[1] not in param_str:
# ... we put for the corresponding index of tab, the value of the parameter as a float.
tab[idx, idy] = abs(float(param_value.replace(',', '.')))
else:
# Otherwise we replace it with a corresponding integer
#tab[idx, idy] = row[2]
param_value = correct_param_value(param_value)
tab[idx, idy] = replaceL[findL.index(param_value)]
np.save(path + '/data/tab_' + str(i) + '.npy', tab)
writer.writerows(tab)
csvfile.close()
output.close()
def correct_param_name(param):
if param == 'FR_déclenchement_IAPB_EC80':
param = 'FR_déclenchement_ IAPB_EC80'
else:
pass
def correct_param_value(param_value):
if param_value == 'Jonctionel':
return 'Jonctionnel'
elif param_value == 'Press. contrôlée':
return 'Pression contrôlée'
elif param_value == 'AI/VS PEP.':
return 'AI/VS PEP'
elif param_value == 'Volume contrôlé.':
return 'Volume contrôlé'
else:
return param_value
def rearrange_files(path, times, params, param_str, n):
max_params = get_max_params(params)
np.save(path + '/data/max_param_' + str(n) + '.npy', max_params)
max_params.insert(0, 'time')
for i in range(24, n):
rearrange_file(path, times, max_params, params, param_str, i)
def get_max_params(params):
max_params = params[0].copy()
for list in params:
for param in list:
if param not in max_params:
max_params.append(param)
max_params.sort()
return max_params
def replace_values(path, i, params):
print("\033[1;36;48m Replacing values in the file ", str(i), '...')
df = pd.read_csv(path + '/data/Signals_' + str(i) + '.csv', delimiter=';')
# Put the veinous pressure positive
if 'P. Veineuse_ECMO' in params[i]:
df['P. Veineuse_ECMO'] = -df['P. Veineuse_ECMO']
for col in param_str:
if col in params[i]:
df[col] = df[col].replace(findL, replaceL)
df.to_csv(path + '/data/Signals_' + str(i) + '.csv', sep=';', index=False, header=False)
def replace_values_all(path, n, params):
for i in range(n):
replace_values(path, i, params)
def interpolate(path, max_params, i):
print("\033[1;37;48m Interpolating missing values in the file ", str(i), '...')
df = pd.read_csv(path + '/data/Signals_' + str(i) + '.csv', delimiter=';')
df.set_index('time', inplace=True)
param_num = [param for param in max_params if param not in param_str]
param_str_in = [param for param in max_params if param in param_str]
df[param_str_in] = df[param_str_in].apply(lambda x: x.interpolate(method='pad', axis=0, limit_direction='both')).astype(float)
df[param_num] = df[param_num].apply(lambda x: x.interpolate(method='index', axis=0, limit_direction='both')).astype(float)
#pd.set_option('display.max_rows', None)
#df.replace('NaN', -1)
#df[max_params] = df[max_params].fillna(-1)
#df.fillna(value=-1, axis=0)
df.to_csv(path + '/data/Signals' + str(i) + '.csv', sep=';', header=False)
def interpolate_all(path, n):
max_params = np.load(path + '/data/max_param_' + str(n) + '.npy', allow_pickle=True)
for i in range(n):
interpolate(path, max_params, i)
"""
j = 1
for time in times[i]:
tab = len(params[i])*[None]
tab[0] = time-time_min
for row in reader:
param = file_params[j]
if row[len(row)-1] == str(time):
if row[1] not in param_str:
tab.append(float(row[2].replace(',', '.')))
else:
tab.append(row[2])
j = j+1
elif row[1] != param and len(tab) < 1 + j:
tab.append(None)
j = j+1
writer.writerow(tab)
csvfile.close()
output.close()
"""
"""
# Re-order each file
for i in range(n):
print(i)
# Open files
csvfile = open('data_CHUV/data_tmp/Signals_10_' + str(i) + '.csv')
output = open('data_CHUV/data/Signals_10_' + str(i) + '.csv', mode='w')
writer = csv.writer(output, delimiter=';')
reader = csv.reader(csvfile, delimiter=';')
# Write header
header = params[i]
header.insert(0, 'time')
writer.writerow(header)
time_min = min(times[i])
file_params = params[i]
print(i)
print(file_params)
# For each time...
for time in times[i]:
csvfile.seek(0)
tab = len(file_params)*['NaN']
tab[0] = time-time_min
# ... we iterate over the rows and we look if the time of the current row is the same as time
for row in reader:
if abs(float(row[-1].replace(',', '.')) - time) <= epsilon:
# If this is the case and that the parameter is not a string...
if row[1] not in param_str:
# ... we put for the corresponding index of tab, the value of the parameter as a float.
tab[file_params.index(row[1])] = float(row[2].replace(',', '.'))
else:
# Otherwise we just put the string as it is in the file
tab[file_params.index(row[1])] = row[2]
writer.writerow(tab)
csvfile.close()
output.close()
"""
c4science · Help