Homec4science
Diffusion Machine Learning ECMO (master) Browse

ML_ECMO/RNN.py
95b55da20f4bmaster

Last Change
Raw File

RNN.py
View Options

import tensorflow as tf
from tensorflow import keras
from tensorflow.compat.v1 import train
from tensorflow.keras.layers import Dense, Flatten, Conv1D, MaxPooling1D, GlobalAveragePooling1D, Dropout, TimeDistributed
from tensorflow.keras.activations import relu, sigmoid, elu, tanh
from tensorflow.keras.layers import BatchNormalization
import tensorflow_probability as tfp
import numpy as np
tf.compat.v1.enable_eager_execution()
class RNN(keras.layers.Layer):
def __init__(self, dim_input, dim_recurrent, dim_output):
super(RNN, self).__init__()
self.fc_x2h_1 = Dense(2*dim_input, input_shape=(dim_input,))
self.fc_x2h_2 = Dense(dim_recurrent, input_shape=(2*dim_input,))
self.fc_h2h_1 = Dense(2*dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2h_2 = Dense(dim_recurrent, input_shape=(2*dim_recurrent,))
self.fc_h2y_1 = Dense(2*dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2y_2 = Dense(dim_output, input_shape=(2*dim_recurrent,))
def call(self, input_data, state):
state = tf.stack(state)
x2h_1 = relu(self.fc_x2h_1(input_data))
h2h_1 = relu(self.fc_h2h_1(state))
h = relu(self.fc_x2h_2(x2h_1) + self.fc_h2h_2(h2h_1))
return sigmoid(self.fc_h2y_2(self.fc_h2y_1(h))), tf.unstack(h)
class RNNReal(keras.layers.Layer):
def __init__(self, dim_input, dim_recurrent, dim_output):
super(RNNReal, self).__init__()
self.fc_x2h_1 = Dense(2*dim_input, input_shape=(dim_input,))
self.fc_x2h_2 = Dense(dim_recurrent, input_shape=(2*dim_input,))
self.fc_h2h_1 = Dense(2*dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2h_2 = Dense(dim_recurrent, input_shape=(2*dim_recurrent,))
self.fc_h2y_1 = Dense(2*dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2y_2 = Dense(dim_output, input_shape=(2*dim_recurrent,))
def call(self, input_data, state):
x2h_1 = relu(self.fc_x2h_1(input_data))
h2h_1 = relu(self.fc_h2h_1(state))
h = relu(self.fc_x2h_2(x2h_1) + self.fc_h2h_2(h2h_1))
return sigmoid(self.fc_h2y_2(self.fc_h2y_1(h))), h
class RNN2Cells(keras.layers.Layer):
def __init__(self, dim_input, dim_recurrent, dim_output):
super(RNN2Cells, self).__init__()
self.fc_x2h_11 = Dense(2*dim_input, input_shape=(dim_input,))
self.fc_x2h_21 = Dense(dim_recurrent, input_shape=(2*dim_input,))
self.fc_x2h_12 = Dense(2 * dim_input, input_shape=(dim_input,))
self.fc_x2h_22 = Dense(dim_recurrent, input_shape=(2 * dim_input,))
self.fc_h2h_11 = Dense(2*dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2h_21 = Dense(dim_recurrent, input_shape=(2*dim_recurrent,))
self.fc_h2h_12 = Dense(2 * dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2h_22 = Dense(dim_recurrent, input_shape=(2 * dim_recurrent,))
self.fc_h2y_1 = Dense(2*dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2y_2 = Dense(dim_output, input_shape=(2*dim_recurrent,))
def call(self, input_data, state):
state = tf.stack(state)
x2h_11 = relu(self.fc_x2h_11(input_data[0]))
h2h_11 = relu(self.fc_h2h_11(state))
h = relu(self.fc_x2h_21(x2h_11) + self.fc_h2h_21(h2h_11))
x2h_12 = relu(self.fc_x2h_12(input_data[1]))
h2h_12 = relu(self.fc_h2h_12(h))
h = relu(self.fc_x2h_22(x2h_12) + self.fc_h2h_22(h2h_12))
return sigmoid(self.fc_h2y_2(self.fc_h2y_1(h))), tf.unstack(h)
class RNN1(keras.layers.Layer):
def __init__(self, dim_input, dim_recurrent, dim_output):
super(RNN1, self).__init__()
self.fc_x2h_1 = Dense(2*dim_input, input_shape=(dim_input,))
self.fc_x2h_2 = Dense(dim_output, input_shape=(2*dim_input,))
self.fc_h2h_1 = Dense(2*dim_recurrent, input_shape=(dim_output,))
self.fc_h2h_2 = Dense(dim_output, input_shape=(2*dim_recurrent,))
def call(self, input_data, state):
state = tf.stack(state)
x2h_1 = relu(self.fc_x2h_1(input_data))
h2h_1 = relu(self.fc_h2h_1(state))
h = relu(self.fc_x2h_2(x2h_1) + self.fc_h2h_2(h2h_1))
return h, tf.unstack(h)
class RNN2(keras.layers.Layer):
def __init__(self, dim_input, dim_recurrent, dim_output):
super(RNN2, self).__init__()
self.fc_x2h_1 = Dense(dim_recurrent, input_shape=(dim_input,))
self.fc_x2h_2 = Dense(dim_input, input_shape=(dim_recurrent,))
self.fc_h2h_1 = Dense(dim_recurrent, input_shape=(dim_input,))
self.fc_h2h_2 = Dense(dim_input, input_shape=(dim_recurrent,))
self.fc_h2y_1 = Dense(dim_recurrent, input_shape=(dim_input,))
self.fc_h2y_2 = Dense(dim_output, input_shape=(dim_recurrent,))
def call(self, input_data, state):
state = tf.stack(state)
x2h_1 = relu(self.fc_x2h_1(input_data))
h2h_1 = relu(self.fc_h2h_1(state))
h = relu(self.fc_x2h_2(x2h_1) + self.fc_h2h_2(h2h_1))
return self.fc_h2y_2(self.fc_h2y_1(h)), tf.unstack(h)
class GatedRNN(keras.layers.Layer):
def __init__(self, dim_input, dim_recurrent, dim_output):
super(GatedRNN, self).__init__()
self.fc_x2h = Dense(dim_recurrent, input_shape=(dim_input,))
self.fc_h2h = Dense(dim_recurrent, input_shape=(dim_recurrent,))
self.fc_x2z = Dense(dim_recurrent, input_shape=(dim_input,))
self.fc_h2z = Dense(dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2y = Dense(dim_output, input_shape=(dim_recurrent,))
def __call__(self, input_data, state):
state = tf.stack(state)
z = sigmoid(self.fc_x2z(input_data) + self.fc_h2z(state))
hb = relu(self.fc_x2h(input_data) + self.fc_h2h(state))
h = z * state + (1 - z) * hb
return self.fc_h2y(h), tf.unstack(h)
class GatedRNN1(keras.layers.Layer):
def __init__(self, dim_input, dim_recurrent, dim_output):
super(GatedRNN1, self).__init__()
self.fc_x2h = Dense(dim_input, input_shape=(dim_input,))
self.fc_h2h = Dense(dim_input, input_shape=(dim_input,))
self.fc_x2z = Dense(dim_recurrent, input_shape=(dim_input,))
self.fc_h2z = Dense(dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2y = Dense(dim_output, input_shape=(dim_recurrent,))
def __call__(self, input_data, state):
state = tf.stack(state)
z = tf.keras.activations.sigmoid(self.fc_x2z(input_data) + self.fc_h2z(state))
hb = self.fc_x2h(input_data) + self.fc_h2h(state)
h = z * state + (1 - z) * hb
return self.fc_h2y(h), tf.unstack(h)
class GatedRNNComplex(keras.layers.Layer):
def __init__(self, dim_input, dim_recurrent, dim_output):
super(GatedRNNComplex, self).__init__()
self.fc_x2h_1 = Dense(dim_recurrent/2, input_shape=(dim_input,))
self.fc_x2h_2 = Dense(dim_recurrent, input_shape=(dim_recurrent/2,))
self.fc_h2h_1 = Dense(2 * dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2h_2 = Dense(dim_recurrent, input_shape=(2 * dim_recurrent,))
self.fc_x2z_1 = Dense(dim_recurrent/2, input_shape=(dim_input,))
self.fc_x2z_2 = Dense(dim_recurrent, input_shape=(dim_recurrent/2,))
self.fc_h2z_1 = Dense(2 * dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2z_2 = Dense(dim_recurrent, input_shape=(2 * dim_recurrent,))
self.fc_h2y_1 = Dense(2 * dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2y_2 = Dense(dim_output, input_shape=(2 * dim_recurrent,))
self.bn1 = BatchNormalization()
self.bn2 = BatchNormalization()
self.bn3 = BatchNormalization()
self.bn4 = BatchNormalization()
self.bn5 = BatchNormalization()
self.bn6 = BatchNormalization()
self.bn7 = BatchNormalization()
self.bn8 = BatchNormalization()
def __call__(self, input_data, state):
state = tf.stack(state)
x2z_1 = self.bn1(relu(self.fc_x2z_1(input_data)))
h2z_1 = self.bn2(relu(self.fc_h2z_1(state)))
z = sigmoid(self.fc_x2z_2(x2z_1) + self.fc_h2z_2(h2z_1))
x2h_1 = self.bn3(relu(self.fc_x2h_1(input_data)))
h2h_1 = self.bn4(relu(self.fc_h2h_1(state)))
hb = relu(self.fc_x2h_2(x2h_1) + self.fc_h2h_2(h2h_1))
h = self.bn5(z) * self.bn6(state) + (1 - z) * self.bn7(hb)
h2y_1 = self.bn8(relu(self.fc_h2y_1(h)))
return self.fc_h2y_2(h2y_1), tf.unstack(h)
class GatedRNNComplex1(keras.layers.Layer):
def __init__(self, dim_input, dim_recurrent, dim_output):
super(GatedRNNComplex1, self).__init__()
self.fc_x2h_1 = Dense(dim_recurrent/2, input_shape=(dim_input,))
self.fc_x2h_2 = Dense(dim_recurrent, input_shape=(dim_recurrent/2,))
self.fc_h2h_1 = Dense(2 * dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2h_2 = Dense(dim_recurrent, input_shape=(2 * dim_recurrent,))
self.fc_x2z_1 = Dense(dim_recurrent/2, input_shape=(dim_input,))
self.fc_x2z_2 = Dense(dim_recurrent, input_shape=(dim_recurrent/2,))
self.fc_h2z_1 = Dense(2 * dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2z_2 = Dense(dim_recurrent, input_shape=(2 * dim_recurrent,))
self.fc_h2y_1 = Dense(2 * dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2y_2 = Dense(dim_output, input_shape=(2 * dim_recurrent,))
self.bn = BatchNormalization(axis=0)
def __call__(self, input_data, state):
state = tf.stack(state)
x2z_1 = relu(self.fc_x2z_1(input_data))
h2z_1 = relu(self.fc_h2z_1(state))
z = sigmoid(self.fc_x2z_2(x2z_1) + self.fc_h2z_2(h2z_1))
x2h_1 = relu(self.fc_x2h_1(input_data))
h2h_1 = relu(self.fc_h2h_1(state))
hb = relu(self.fc_x2h_2(x2h_1) + self.fc_h2h_2(h2h_1))
h = z * state + (1 - z) * hb
h2y_1 = relu(self.fc_h2y_1(h))
return self.fc_h2y_2(h2y_1), tf.unstack(h)
class GatedRNNMaster(keras.layers.Layer):
def __init__(self, dim_input, dim_recurrent, dim_output):
super(GatedRNNMaster, self).__init__()
self.fc_x2h_1 = Dense(dim_recurrent/2, input_shape=(dim_input,))
self.fc_x2h_2 = Dense(2 * dim_recurrent, input_shape=(dim_recurrent/2,))
self.fc_x2h_3 = Dense(dim_recurrent, input_shape=(dim_recurrent / 2,))
self.fc_h2h_1 = Dense(2 * dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2h_2 = Dense(3 * dim_recurrent, input_shape=(2 * dim_recurrent,))
self.fc_h2h_3 = Dense(dim_recurrent, input_shape=(3 * dim_recurrent,))
self.fc_x2z_1 = Dense(dim_recurrent/2, input_shape=(dim_input,))
self.fc_x2z_2 = Dense(2*dim_recurrent, input_shape=(dim_recurrent/2,))
self.fc_x2z_3 = Dense(dim_recurrent, input_shape=(2*dim_recurrent,))
self.fc_h2z_1 = Dense(2 * dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2z_2 = Dense(3 * dim_recurrent, input_shape=(2 * dim_recurrent,))
self.fc_h2z_3 = Dense(dim_recurrent, input_shape=(3 * dim_recurrent,))
self.fc_h2y_1 = Dense(2 * dim_recurrent, input_shape=(dim_recurrent,))
self.fc_h2y_2 = Dense(3 * dim_recurrent, input_shape=(2 * dim_recurrent,))
self.fc_h2y_3 = Dense(dim_output, input_shape=(3 * dim_recurrent,))
def __call__(self, input_data, state):
state = tf.stack(state)
x2z_1 = elu(self.fc_x2z_1(input_data))
x2z_2 = elu(self.fc_x2z_2(x2z_1))
h2z_1 = elu(self.fc_h2z_1(state))
h2z_2 = elu(self.fc_h2z_2(h2z_1))
z = sigmoid(self.fc_x2z_3(x2z_2) + self.fc_h2z_3(h2z_2))
x2h_1 = elu(self.fc_x2h_1(input_data))
x2h_2 = elu(self.fc_x2h_2(x2h_1))
h2h_1 = elu(self.fc_h2h_1(state))
h2h_2 = elu(self.fc_h2h_2(h2h_1))
hb = elu(self.fc_x2h_3(x2h_2) + self.fc_h2h_3(h2h_2))
h = z * state + (1 - z) * hb
h2y_1 = elu(self.fc_h2y_1(h))
h2y_2 = elu(self.fc_h2y_2(h2y_1))
return self.fc_h2y_3(h2y_2), tf.unstack(h)
def train_model(model, train_input, train_target, states, learning_rate, num_epochs=25, mini_batch_size=10, loss_tab=[]):
optimizer = train.AdamOptimizer(learning_rate=learning_rate)
mse = tf.keras.losses.MeanSquaredError()
#mse = tf.keras.losses.mean_squared_logarithmic_error()
global_step = tf.Variable(0)
for epoch in range(num_epochs):
#perm = np.random.permutation(500)
#train_input = train_input[perm, :, :]
#train_target = train_target[perm, :, :]
#states = list(train_target[i, 0, :] for i in range(500))
states = list(tf.zeros(shape=[states[0].shape[0]]) for i in range(train_input.shape[0]))
for b in range(0, train_input.shape[0], mini_batch_size):
loss_value = 0
for t in range(train_input.shape[1]):
with tf.GradientTape() as tape:
output, states[b:b + mini_batch_size] = model(train_input[b:b + mini_batch_size, t], states[b:b + mini_batch_size])
loss_value += mse(output, train_target[b:b + mini_batch_size, t])
#loss_tab.append(loss_value)
grads = tape.gradient(loss_value, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables), global_step)
loss_tab.append(loss_value/31)
print("Epoch : ", epoch)
def train_model_real(model, train_input, train_target, NAN_targets, dim_recurrent, learning_rate, num_epochs=25, loss_tab=[]):
optimizer = train.AdamOptimizer(learning_rate=learning_rate)
mse = tf.keras.losses.MeanSquaredError()
global_step = tf.Variable(0)
for epoch in range(num_epochs):
for i in range(len(train_input)):
loss_value = 0
state = tf.zeros(shape=[1, dim_recurrent], dtype=tf.float64)
n = train_input[i].shape[1]//30
c = 0
for t in range(train_input[i].shape[1]):
with tf.GradientTape() as tape:
output, state = model(train_input[i][:, t], state)
loss_value += mse(tf.boolean_mask(output[0], tf.logical_not(NAN_targets[i][0, t])),
tf.boolean_mask(train_target[i][0, t],
tf.logical_not(NAN_targets[i][0, t])))
c = c + 1.0
if t % n == 0 or t == train_input[i].shape[1]-1:
grads = tape.gradient(loss_value, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables), global_step)
c = c + 1.0
loss_tab.append(loss_value / c)
#print(c)
print(loss_value)
#loss_tab.append(loss_value/train_input[i].get_shape().as_list()[1])
print("Epoch : ", epoch)
def train_model_2cells(model, train_input, train_target, states, learning_rate, num_epochs=25, mini_batch_size=10, loss_tab=[]):
optimizer = train.AdamOptimizer(learning_rate=learning_rate)
mse = tf.keras.losses.MeanSquaredError()
#mse = tf.keras.losses.mean_squared_logarithmic_error()
global_step = tf.Variable(0)
for epoch in range(num_epochs):
#perm = np.random.permutation(500)
#train_input = train_input[perm, :, :]
#train_target = train_target[perm, :, :]
#states = list(train_target[i, 0, :] for i in range(500))
states = list(tf.zeros(shape=[states[0].shape[0]]) for i in range(1500))
for b in range(0, train_input.shape[0], mini_batch_size):
loss_value = 0
for t in range(1, train_input.shape[1]):
with tf.GradientTape() as tape:
input_data = list(train_input[b:b + mini_batch_size, i] for i in range (t-1,t+1))
output, states[b:b + mini_batch_size] = model(input_data, states[b:b + mini_batch_size])
loss_value += mse(output, train_target[b:b + mini_batch_size, t])
grads = tape.gradient(loss_value, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables), global_step)
loss_tab.append(loss_value/51)
print(epoch)
def compute_loss(model, test_input, test_target, states, num_epochs=25, mini_batch_size=10, loss_tab=[]):
mse = tf.keras.losses.MeanSquaredError()
states = list(tf.zeros(shape=[64]) for i in range(500))
predictions = []
predictions_tmp = []
for b in range(0, test_input.shape[0], mini_batch_size):
loss_value = 0
predictions_tmp = [test_target[b:b + mini_batch_size, 0, :]]
for t in range(test_input.shape[1]):
output, states[b:b + mini_batch_size] = model(test_input[b:b + mini_batch_size, t],
states[b:b + mini_batch_size])
prediction, state = model(test_input[b:b + mini_batch_size, t + 1, :], states[b:b + mini_batch_size])
predictions_tmp.append(prediction)
loss_value += mse(output, test_target[b:b + mini_batch_size, t])
predictions_tmp = tf.stack(predictions_tmp, axis=1)
predictions.append(predictions_tmp)
loss_tab.append(loss_value // test_input.shape[1])
predictions = tf.stack(predictions_tmp, axis=0)
return predictions
def tempconvnet(offset, dim_input, dim_output, loss, optimizer):
convmodel = keras.Sequential([
Conv1D(64, 2, activation='relu', input_shape=(offset, dim_input)),
#MaxPooling1D(2),
Conv1D(32, 2, activation='relu'),
#MaxPooling1D(2),
#Conv1D(16, 2, activation='relu'),
GlobalAveragePooling1D(),
Dropout(0.5),
Dense(dim_output, activation='relu')])
convmodel.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
return convmodel
def train_tempconvnet(convnet, train_input, train_target, time_steps, offset, batch_size, nb_epochs):
for t in range(time_steps-offset):
convnet.fit(train_input[:, t:offset+t], train_target[:, offset+t], batch_size=batch_size, epochs=nb_epochs)
def tempconvnet_real(offset, dim_input, dim_output, loss, optimizer):
convmodel = keras.Sequential([
Conv1D(2*dim_input, 3, activation='relu', input_shape=(offset, dim_input)),
MaxPooling1D(3),
Conv1D(3*dim_input, 3, activation='relu'),
MaxPooling1D(3),
Conv1D(2*dim_output, 3, activation='relu'),
GlobalAveragePooling1D(),
Dropout(0.5),
Dense(dim_output, activation='relu')])
convmodel.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
return convmodel
def train_tempconvnet_real(convnet, train_input, train_target, time_steps, offset, batch_size, nb_epochs):
for t in range(time_steps-offset):
convnet.fit(train_input[:, t:offset+t], train_target[:, offset+t], batch_size=batch_size, epochs=nb_epochs)
c4science · Help