Page MenuHomec4science
Files F120489326

compute-run-variance.sh
No OneTemporary
Actions

File Metadata

Created
Fri, Jul 4, 18:06

compute-run-variance.sh
View Options

#!/bin/bash
YARN_PATH=/HDD-2TB/YARN
if [ $# -ne 1 ]; then
echo "$0 <dir>"
exit 1
fi
cd $1
JOB_FILES=$(ls *all.csv.files)
#declare -A JOB_TIMES
NUM_JOBS=$(cat TRACE*.trace | grep job.id | wc -l)
for JOB_FILE in $JOB_FILES; do
#
# echo $JOB_FILE
#
# unset JOB_TIMES
# JOB_NAMES=""
# for LINE in $(cat $JOB_FILE | sort -h); do
# echo $LINE | grep -q "JobID" && continue
#
# IFS=', ' read -r -a array <<< "$(echo $LINE | cut -d',' -f1,5,6)"
#
# end=${array[2]}
# beg=${array[1]}
# [ "${JOB_TIMES[${array[0]}]}" == "" ] && JOB_NAMES="$JOB_NAMES ${array[0]}"
# JOB_TIMES[${array[0]}]="${JOB_TIMES[${array[0]}]} $((end-beg))"
#
# done
#
# for JOB_NAME in $JOB_NAMES; do
# echo ${JOB_TIMES[$JOB_NAME]} | $YARN_PATH/scripts/stats | grep COV | cut -d':' -f2
# done | $YARN_PATH/scripts/stats | grep max | cut -d':' -f2
split -l $((NUM_JOBS + 1)) $JOB_FILE
for SPLIT in x*; do
cat $SPLIT | tail -n +2 | awk -F',' '{ print $6-$5 }' | sort -n > ${SPLIT}.jrt
$YARN_PATH/scripts/cdf-gen.py ${SPLIT}.jrt 0 > ${SPLIT}.jrt.cdf
cat ${SPLIT}.jrt.cdf | cut -f2 > ${SPLIT}.jrt.cdf.1col
done
paste x*.jrt.cdf.1col | column -s $'\t' -t > ${JOB_FILE%-all.csv.files}.jrt.cdfs
cat ${JOB_FILE%-all.csv.files}.jrt.cdfs | xargs -L1 sh -c 'echo "$@" | tr " " "\n" | '$YARN_PATH'/scripts/stats 2>/dev/null | grep COV | tr -s " " | cut -d" " -f3' > ${JOB_FILE%-all.csv.files}.covs
cat xaa.jrt.cdf | cut -f1 > xaa.jrt.cdf.1col
paste xaa.jrt.cdf.1col ${JOB_FILE%-all.csv.files}.jrt.cdfs ${JOB_FILE%-all.csv.files}.covs > ${JOB_FILE%-all.csv.files}.stats
paste xaa.jrt.cdf.1col ${JOB_FILE%-all.csv.files}.covs > ${JOB_FILE%-all.csv.files}.sum_stats
rm -f x*
done
cd - &>/dev/null

Event Timeline

c4science · Help