Homec4science
Diffusion elastic-yarn (future) Browse

R3704/scripts/exp-runner.sh
f786c6ebb891future

Last Change
Raw File

exp-runner.sh
View Options

#!/bin/bash
[[ -z $SCRIPT_CONFIG_PATH ]] && echo "SCRIPT_CONFIG_PATH not set!" && exit 1
. $SCRIPT_CONFIG_PATH/config_path.sh
. $SCRIPT_CONFIG_PATH/config_classpath.sh
if [ $# -ne 13 ]
then
echo "Wrong number of parameters given. Given $#"
echo "Usage: $0 JOB_FILE TOPO_FILE SIM_TYPE WAIT_ESTIMATION PENALTY_TYPE MIN_RATIO PORT_START HB_INTERVAL HB_SEED CORES INITIAL_BUMP EXPID CTR"
exit -1
fi
JOB_FILE=$1
shift
TOPO_FILE=$1
shift
SIM_TYPE=$1
shift
WAIT_ESTIMATION=$1
shift
PENALTY_TYPE=$1
shift
MIN_RATIO=$1
shift
PORT_START=$1
shift
HB_INTERVAL=$1
shift
HB_SEED=$1
shift
CORES=$1
shift
INITIAL_BUMP=$1
shift
EXPID=$1
shift
ctr=$1
shift
if [ $SIM_TYPE != 'greedy' ] && [ $SIM_TYPE != 'online' ] && [ $SIM_TYPE != 'informed' ] && [ $SIM_TYPE != 'reserved' ] && [ $SIM_TYPE != 'regular' ]; then
echo "Invalid sim-type chosen: $SIM_TYPE. Valid options are: 'greedy', 'online', 'informed', 'reserved' or 'regular'"
exit -1
fi
if [ $WAIT_ESTIMATION != 'hAgg' ] && [ $WAIT_ESTIMATION != 'hInd' ] && [ $WAIT_ESTIMATION != 'head' ]&& [ $WAIT_ESTIMATION != 'current' ] && [ $WAIT_ESTIMATION != 'future' ]; then
echo "Invalid prediction type chosen: $WAIT_ESTIMATION. Valid options are: 'hAgg', 'hInd', 'head', 'current' or 'future'"
exit -1
fi
if [ $PENALTY_TYPE != 'power1' ] && [ $PENALTY_TYPE != 'power2' ] && [ $PENALTY_TYPE != 'squareroot' ] && [ $PENALTY_TYPE != 'sawtooth' ] && [ $PENALTY_TYPE != 'step' ] && [ $PENALTY_TYPE != 'none' ]; then
echo "Invalid penalty type chosen: $PENALTY_TYPE. Valid options are: 'power1', 'power2', 'squareroot', 'sawtooth', 'step' or 'none'"
exit -1
fi
BASE_DIR=$YARN_SLS_RUN_PATH
EXP_DIR=$YARN_SLS_RUN_PATH/xexperiments
exp () {
PARAM_STRING=""
[ "$SIM_TYPE" == "informed" ] && PARAM_STRING="$PARAM_STRING---$WAIT_ESTIMATION"
if [ "$SIM_TYPE" == "informed" ] || [ "$SIM_TYPE" == "greedy" ] || [ "$SIM_TYPE" == "online" ]; then
PARAM_STRING="$PARAM_STRING---$PENALTY_TYPE"
if [ "$PENALTY_TYPE" == "sawtooth" ] || [ "$PENALTY_TYPE" == "step" ]; then
PARAM_STRING="$PARAM_STRING---IB_$INITIAL_BUMP"
fi
fi
OUT_DIR=$EXP_DIR/$EXPID-$ctr-$(basename $JOB_FILE)---$(basename $TOPO_FILE)---${SIM_TYPE}${PARAM_STRING}---`date +"%Y-%m-%d_%H-%M-%S"`
TRUNC=$EXP_DIR/$EXPID-$ctr-${SIM_TYPE}$(echo $PARAM_STRING | tr -s '-')
echo "Output dir $OUT_DIR"
mkdir -p $OUT_DIR
cd $BASE_DIR
YARN_HEAPSIZE=10000
HADOOP_HEAPSIZE=10000
export YARN_HEAPSIZE
export HADOOP_HEAPSIZE
PATH="$PATH:$YARN_PATH/hadoop-2.3.0/bin" \
HADOOP_CLASSPATH="${HADOOP_CLASSPATH}" \
JAVA_HOME="${JAVA_HOME}" \
bin/slsrun.sh \
--input-sls=$JOB_FILE \
--output-dir=$OUT_DIR \
--nodes=$TOPO_FILE \
--print-simulation \
-OhbInterval=$HB_INTERVAL \
-Ohbseed=$HB_SEED \
-Ocores=$CORES \
-OinitialBump=$INITIAL_BUMP \
-Osimtype=$SIM_TYPE \
-OwaitTimeEstimation=$WAIT_ESTIMATION \
-Openalty=$PENALTY_TYPE \
-OminRatio=$MIN_RATIO \
-Oportstart=$PORT_START &> $OUT_DIR/$(basename ${TRUNC})-simulation.log
cp $TOPO_FILE $JOB_FILE $OUT_DIR
cd -
cd $OUT_DIR
echo "Done with $TRUNC"
grep app jobruntime.csv | awk -F, '{print $1,$6-$5}' | sed 's/_/ /g' | awk '{print $2,$3/1000}' | sed 's/^0*//g' > $TRUNC.jrt
$YARN_PATH/scripts/cdf-gen.py $TRUNC.jrt 1 > $TRUNC.jrt.cdf
# Parse log file for informed wait-related info
# if [ $SIM_TYPE == "informed" ]; then
# $YARN_PATH/scripts/parse_est_times.py ${TRUNC}-simulation.log $TRUNC
# for suffix in $(echo "dcr dpcr ddr cgt cgm cgmd"); do
# $YARN_PATH/scripts/cdf-gen.py $TRUNC.$suffix 0 > $TRUNC.$suffix.cdf
# done
# fi
cat metrics/variable.cluster.allocated.memory.csv | sed 's/,/ /g' | grep -v value | awk 'BEGIN{s=0}{s++; print s,$2/1000}' > $TRUNC.tmem
cd -
ctr=$(($ctr+1))
}
exp
c4science · Help