Page MenuHomec4science
Files F112262275

create_sbatch.sh
No OneTemporary
Actions

File Metadata

Created
Fri, May 9, 10:48

create_sbatch.sh
View Options

#!/bin/bash
if [ "$1" == "-h" ] ; then
echo ""
echo "####################################################"
echo "This script replaces lines in a template file (submit_template.sh) and create a workspace for the jobs submission."
echo "You need to specify/have:"
echo "- a template file 'submit_template.sh' in the directory where you are executing the script"
echo "- a python executable file in your working directory"
echo "- a set of inputs for your jobs, meaning: model, partition (CPU or GPU), memory, input data folder, test dataset, nr. nodes for the ML algorithm."
echo "By invoking this script you will create a workspace with files having the righ formatting for the submission on sbatch system."
echo "Author: roberto.castello@epfl.ch"
echo "Date: 19/03/2018 "
echo "####################################################"
echo ""
exit 0
fi
## STATIC inputs
TEMPLATE_FILE="submit_template.sh"
GPU_syntax="#SBATCH --partition=gpu --qos=gpu_free --gres=gpu:1"
## CUSTOMIZABLE inputs
PYTHON_EXECUTABLE="HPELM_RC"
MYDIR="energy-potential\/Solar"
MODEL="ELM"
PART_VEC=("GPU" "CPU")
MEM="4G" ## "500M"
DATA_VEC=("2012-2012_grid100_SIS" "2012-2012_grid500_SIS" "2012-2015_grid100_SIS" "2012-2015_grid500_SIS")
QUERY="grid1600"
NODE_VEC=("500" "1000" "2000" "3000")
for DATA in ${DATA_VEC[@]}
do
echo "[MSG] Creating workspace $DATA"
echo ""
## == Creating dir if not already existing and copying the python executable into it
if [ ! -d workspace_$MODEL\_$DATA\_$QUERY ]
then
mkdir workspace_$MODEL\_$DATA\_$QUERY
mkdir workspace_$MODEL\_$DATA\_$QUERY/output
fi
## == Printing lines of the template (can be skipped)
#cat $TEMPLATE_FILE | while read line_input
#do
# [[ $line_input = \#\#* ]] && continue
# echo $line_input
#done
for PART in ${PART_VEC[@]}
do
for NODE in ${NODE_VEC[@]}
do
echo "..."
cp $TEMPLATE_FILE workspace_$MODEL\_$DATA\_$QUERY\/submit_$PART\_$MEM\_$NODE\_tmp.sh
## == Replacing the custom choice in a temporary file
sed -i "s/\_mem/$MEM/g" workspace_$MODEL\_$DATA\_$QUERY\/submit_$PART\_$MEM\_$NODE\_tmp.sh
sed -i "s/\_mydir/$MYDIR/g" workspace_$MODEL\_$DATA\_$QUERY\/submit_$PART\_$MEM\_$NODE\_tmp.sh
sed -i "s/\_python/$PYTHON_EXECUTABLE/g" workspace_$MODEL\_$DATA\_$QUERY\/submit_$PART\_$MEM\_$NODE\_tmp.sh
sed -i "s/\_model/$MODEL/g" workspace_$MODEL\_$DATA\_$QUERY\/submit_$PART\_$MEM\_$NODE\_tmp.sh
sed -i "s/\_data/$DATA/g" workspace_$MODEL\_$DATA\_$QUERY\/submit_$PART\_$MEM\_$NODE\_tmp.sh
sed -i "s/\_query/$QUERY/g" workspace_$MODEL\_$DATA\_$QUERY\/submit_$PART\_$MEM\_$NODE\_tmp.sh
sed -i "s/\_node/$NODE/g" workspace_$MODEL\_$DATA\_$QUERY\/submit_$PART\_$MEM\_$NODE\_tmp.sh
echo ".."
## == Adding the GPU syntax (if needed)
if [ $PART == GPU ]
then
sed -i "s/\_part/$GPU_syntax/" workspace_$MODEL\_$DATA\_$QUERY\/submit_$PART\_$MEM\_$NODE\_tmp.sh
else
sed -i 's/\_part/\#\#/' workspace_$MODEL\_$DATA\_$QUERY\/submit_$PART\_$MEM\_$NODE\_tmp.sh
fi
## == Copying the temporary file to the final one and cleaning
cp workspace_$MODEL\_$DATA\_$QUERY\/submit_$PART\_$MEM\_$NODE\_tmp.sh workspace_$MODEL\_$DATA\_$QUERY\/$PART\_$MEM\_$NODE\.sh
rm workspace_$MODEL\_$DATA\_$QUERY\/submit_$PART\_$MEM\_$NODE\_tmp.sh
echo "."
echo ""
done
done
## == Creating a script (run.sh) in the workspace for the sbatch submission (N.B. not executing, just creating)
cd workspace_$MODEL\_$DATA\_$QUERY\/
suffix=".sh"
if [ ! -f run.sh ]; then
echo "#!/bin/bash" >> run.sh
chmod +x run.sh
fi
for f in ./*_*.sh; do
name=$(basename "$f")
echo "sbatch -J" ${name%$suffix} $name >> run.sh
done
cd ../
## == Terminating
echo "[DONE] The file(s) "$PART"_"$MEM"_<nodes>.sh and relative script for the submission on batch queues have been created inside workspace_"$MODEL"_"$DATA"_"$QUERY"/"
done

Event Timeline

c4science · Help