Homec4science
Diffusion solarPV (aw) Browse

solarPV_identification/CNN/model_main.py
035f8d3132e6aw

Last Change
Raw File

model_main.py
View Options

import os
import tensorflow as tf
import numpy as np
import scipy
import data_loaders
import datetime
import losses
import sys
import json
import itertools
import codecs
import fcntl
import argparse
from mpi_routines import *
from filelock import FileLock
from mpi4py import MPI
import horovod.keras as hvd
#import cv2
from scipy import ndimage
#from matplotlib import pyplot as plt
from PIL import Image
from keras.models import *
from keras.layers import Reshape
from keras.layers.core import *
from keras.layers import *
from keras.layers.normalization import BatchNormalization
from keras.layers.convolutional import *
from keras.layers.advanced_activations import LeakyReLU
from keras.optimizers import Adam , SGD
from keras.utils import np_utils
from io import StringIO
from keras import backend as K
from keras.callbacks import CSVLogger, ModelCheckpoint
def unet(nClasses, loss, optimizer=None, input_width=360, input_height=480, nChannels=1,
dropout_rate=0.2):
ker = 3
pad = 3
inputs = Input((input_height, input_width, nChannels))
inputs_padded = ZeroPadding2D(padding=pad, input_shape=(input_width, input_height, 3))(inputs)
conv1 = Conv2D(32, kernel_size=ker, activation='relu', padding='same')(inputs_padded)
conv1 = Dropout(dropout_rate)(conv1)
conv1 = Conv2D(32, kernel_size=ker, activation='relu', padding='same')(conv1)
pool1 = MaxPooling2D(padding='same')(conv1)
conv2 = Conv2D(64, kernel_size=ker, activation='relu', padding='same')(pool1)
conv2 = Dropout(dropout_rate)(conv2)
conv2 = Conv2D(64, kernel_size=ker, activation='relu', padding='same')(conv2)
pool2 = MaxPooling2D(padding='same')(conv2)
conv3 = Conv2D(128, kernel_size=ker, activation='relu', padding='same')(pool2)
conv3 = Dropout(dropout_rate)(conv3)
conv3 = Conv2D(128, kernel_size=ker, activation='relu', padding='same')(conv3)
up1 = concatenate([UpSampling2D(size=(2, 2))(conv3), conv2], axis=3)
conv4 = Conv2D(64, kernel_size=ker, activation='relu', padding='same')(up1)
conv4 = Dropout(dropout_rate)(conv4)
conv4 = Conv2D(64, kernel_size=ker, activation='relu', padding='same')(conv4)
up2 = concatenate([UpSampling2D(size=(2, 2))(conv4), conv1], axis=3)
conv5 = Conv2D(32, kernel_size=ker, activation='relu', padding='same')(up2)
conv5 = Dropout(dropout_rate)(conv5)
conv5 = Conv2D(32, kernel_size=ker, activation='relu', padding='same')(conv5)
conv6 = Conv2D(nClasses, kernel_size=1, activation='relu', padding='same')(conv5)
cropped_conv6 = Cropping2D(cropping=pad)(conv6)
conv6 = core.Reshape((input_height * input_width, nClasses))(cropped_conv6)
conv7 = core.Activation('softmax')(conv6)
model = Model(inputs=inputs, outputs=conv7)
if not optimizer is None:
model.compile(loss=loss, optimizer=optimizer, metrics=[losses.dice_coef]) ## metric: here using IoU
print("Model was compiled.")
return model
def run(hyperparams, directory_path, hvd_size, hvd_rank):
paramfile = data_loaders.read_json()
rotations = paramfile['rotations']
lights = paramfile['lights']
dice = paramfile['dice']
weight1 = paramfile['weight1']
weight2 = paramfile['weight2']
print(hyperparams, 'BBBBB')
epochs = int(hyperparams['epochs'])
dropout = float(hyperparams['dropout'])
loss = hyperparams['loss']
learning_rate = hyperparams['learning_rate']
loss_dict = {'dice': losses.dice_coef_loss,
'wcd': losses.weighted_categorical_crossentropy([weight1, weight2])}
optimizer = hyperparams['optimizer']
optimizer_dict = {'adam' : Adam(lr=float(learning_rate)*hvd_size),
'sgd' : SGD(lr=float(learning_rate)*hvd_size)}
original_images, labeled_images = data_loaders.load_images(lights=lights, rotations=rotations)
X_train = np.array(original_images)
Y_train_flat = np.array(labeled_images).reshape(len(labeled_images), -1)
# One dimension for each channel (one channel PV, one channel non-PV)
Y_train_2d = np.stack((np.where(Y_train_flat == 0, 1, 0), Y_train_flat), axis=-1).reshape(len(X_train), -1, 2)
inp_shape = X_train[0].shape
out_shape_flat = Y_train_flat[0].shape
out_shape_2d = Y_train_2d[0].shape
optimizer = optimizer_dict[optimizer]
optimizer = hvd.DistributedOptimizer(optimizer)
model_u = unet(2, loss=loss_dict[loss],optimizer=optimizer, input_width=250,
input_height=250, nChannels=3, dropout_rate=dropout)
model_u.summary()
model_name = str(datetime.date.today()) +\
"_d{:}_w{:}_e{:}_rl{:}".format(dice, (weight1, weight2),
epochs, (rotations, lights))
print(model_name)
epochs = int(hyperparams['epochs'])
dropout = float(hyperparams['dropout'])
loss = hyperparams['loss']
learning_rate = hyperparams['learning_rate']
loss_dict = {'dice': losses.dice_coef_loss,
'wcd': losses.weighted_categorical_crossentropy([weight1, weight2])}
csv_name = "e_{:}_d{:}_{:}_lr{:}_{:}".format(epochs, dropout,
loss, learning_rate, optimizer)
csv_logger = CSVLogger(csv_name+'.csv', append=True, separator=' ')
callbacks=[csv_logger, hvd.callbacks.BroadcastGlobalVariablesCallback(0)]
#if hvd_rank == 0:
# callbacks.append(ModelCheckpoint(directory_path + './checkpoint-{epoch}.h5'))
model_u.fit(X_train, Y_train_2d, validation_split=0.1, epochs=epochs,
callbacks=callbacks)
# Save the weights
if hvd_rank == 0:
model_u.save(directory_path + '{:}.h5'.format(model_name))
# Save the weights
model_u.save_weights(directory_path + '{:}weights.h5'.format(model_name))
# Save the model architecture
with open(directory_path +'{:}architecture.json'.format(model_name), 'w') as f:
f.write(model_u.to_json())
with open(directory_path +'{:}parameters.json'.format(model_name), 'w') as f:
json.dump(paramfile, f)
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--restart_read', type=str, help='Read the restart file')
parser.add_argument('--restart_write', type=str, help='Write the restart file')
parser.add_argument('--nreplica', type=int, help='Number of replica for hyperparameter tuning. Each replica will be used by horovod.')
args = parser.parse_args()
return args
if __name__ == "__main__":
restart_read = parse_args().restart_read
restart_write = parse_args().restart_write
nreplica = parse_args().nreplica
intra_comm = set_intra_comm(nreplica)
inter_comm = set_inter_comm(nreplica)
params = distribute_parameters(inter_comm, intra_comm, nreplica,
restart_file=restart_read)
directory_path = params['dir_path']
set_horovod(inter_comm, intra_comm)
#with open('original', 'w') as f:
# json.dump(params['hyperparam'], f)
open('hyper_done', 'w+').close()
for i, h in enumerate(params['hyperparam']):
run(h, directory_path, intra_comm.size, intra_comm.rank)
global_index = int(params['hyper_index'][intra_comm.rank-1]+i) if intra_comm.rank>0 else i
with FileLock(restart_write):
with open('hyper_done', 'a+') as f:
json.dump({'hyper_index': global_index,'hyper_val':
params['hyperparam'][i]}, f)
restart_params = np.delete(params['hyperparam'], i, axis=0)
with open(restart_write, 'w+') as f:
json.dump(restart_params.tolist(), f)
#json.dump(params['hyperparam'], codecs.open(directory_path + str(world_rank)
# + 'hyper.json', 'w', encoding='utf-8'), separators=(',', ':'),
# sort_keys=True, indent=4)
c4science · Help