#!/bin/bash -e

#
# Author: Alfio Lazzaro, alazzaro@cray.com (2013)
# Script to generate LIBSMM library
# Run ./generate -h to see the help
#
echo
echo "Script to generate LIBSMM library."
echo "Author: Alfio Lazzaro, alazzaro@cray.com (2013)"
echo

#
# Source the library with all routines
#
source generate.bash

#
# Default OPTIONS values
#
def_config_dir="config"
def_config_file="${def_config_dir}/cray.gnu"
def_SIMD="avx"
def_ntasks="16"
def_base_out_dir="output"
def_target="all"
def_njobs="1"
def_wlm="slurm"
def_wtime="01:00:00"

show_help() {
    echo "Run with: ./generate [OPTIONS] [COMMAND]"
    echo
    echo "OPTIONS are:"
    echo "   -h               : show this help."
    echo "   -c <config file> : config file to use (declared inside ${def_config_dir} directory)."
    echo "                      Default value is \"${def_config_file}\"."
    echo "   -j <#>           : set the number of jobs for batch submission. Setting to zero means no batch submission."
    echo "                      Default value is \"${def_njobs}\"."
    echo "   -s <SIMD type>   : SIMD registers type (sse, avx, mic). Use novec to not generate vector kernel."
    echo "                      Default value is \"${def_SIMD}\"."
    echo "   -t <#>           : set the number of tasks per each node."
    echo "                      Default value is ${def_ntasks}."
    echo "   -w <wlm type>    : workload manager for batch submission. The value must correspond to one of the files ${def_config_dir}/*.wlm."
    echo "                      Default value is \"${def_wlm}\"."
    echo "   -m               : time limit for batch execution."
    echo "                      Default value is \"${def_wtime}\"."
    echo
    echo "COMMAND is one of the followings:"
    echo "   tiny1  : it runs the tiny phase. Batch execution if requested."
    echo "   tiny2  : collect the results of the tiny phase."
    echo "            Run automatically during tiny1 when there is no batch execution."
    echo "   small1 : it runs the small phase. Batch execution if requested."
    echo "   small2 : collect the results of the small phase. "
    echo "            Run automatically during small1 when there is no batch execution."
    echo "   lib    : generate the libray. Batch execution if requested (a single job needed)."
    echo "   check1 : run the checks on the library. Batch execution if requested."
    echo "   check2 : collect the results of the checks."
    echo "            Run automatically during check1 when there is no batch execution."
    echo "            NOTE: same number of jobs used in check1 must be used for check2."
    echo "NOTE: COMMANDs must be executed in the above order."
    echo
    echo "Special COMMAND:"
    echo "    clean : remove intermediate files (but not some key output and the library itself)."
    echo " cleanall : remove all intermediate files."
    echo
    exit
}

#
# Load OPTIONS
#
check_number()
{
    if ! [[ "$1" =~ ^[0-9]+$ ]] ; then
	echo "Error: $1 is not a valid number for -$2 option!."
	echo "Run ./generate -h for help."
	exit
    fi

    # Check for positive numbers
    if [ $# -gt 2 -a $1 -le 0 ]; then
	echo "Error: $1 must be positive for -$2 option!."
	echo "Run ./generate -h for help."
	exit
    fi

}

while getopts "c:hj:s:t:w:m:" OPTION; do
    case $OPTION in
	c)
	    config_file=$OPTARG
	    ;;  
	h)
	    show_help
	    ;;
	j)
	    jobs=$OPTARG
	    check_number $jobs $OPTION
	    ;;
	s)
	    SIMD=$OPTARG
	    ;;	    
	t)
	    ntasks=$OPTARG
	    check_number $ntasks $OPTION $ntasks
	    ;;
	w)
	    wlm=$OPTARG
	    ;;
	m)
	    wtime=$OPTARG
	    ;;
        ?) 
            exit
            ;;
    esac
done
shift $(( OPTIND - 1))

#
# Use default OPTIONS values if they were not declared
#
if [ -z "${config_file}" ]; then
    config_file=$def_config_file
fi

if [ -z "${SIMD}" ]; then
    SIMD=$def_SIMD
fi

if [ -z "${ntasks}" ]; then
    ntasks=$def_ntasks
fi

if [ -z "${jobs}" ]; then
    jobs=${def_njobs}
fi

if [ -z "${wlm}" ]; then
    wlm=${def_wlm}
fi

if [ -z "${wtime}" ]; then
    wtime=${def_wtime}
fi

base_out_dir=$def_base_out_dir
target=$def_target

#
# Check COMMAND value
#
if [ $# -eq 0 ]; then
    echo "Missing COMMAND! Run ./generate -h for help."
    exit
fi

cmd=`echo $1 | awk '{ print tolower($0)}'`

case $cmd in
    tiny1|tiny2|small1|small2|lib|check1|check2)
	;;
    clean|cleanall)
	echo "Remove files."
	rm -fR run_tiny_*/
	rm -fR run_small_*/
	rm -fR run_lib_*/
	rm -f *.mod Makefile.* *.x *.o
	rm -f *~
	if [ "$cmd" == "cleanall" ]; then
	    rm -f *.out
	    rm -fR lib
	fi
	echo
	exit
	;;
    *)
	echo "Unknown COMMAND \"$1\". Run ./generate -h for help."
	exit
	;;
esac

#
# Check OPTIONS values
#
if [ ! -e ${config_file} ]; then
    echo "Error: config file \"${config_file}\" doesn't exist!"
    echo "Available config files are:"
    ls ${def_config_dir} | grep -v "wlm"
    exit
fi

if [ ! -e ${def_config_dir}/${wlm}.wlm ]; then
    echo "Error: corresponding wlm file to \"${wlm}\" (${wlm}.wlm) doesn't exist inside ${def_config_dir} directory!"
    echo "Available wlm files are:"
    ls ${def_config_dir} | grep "wlm" | cut -d'.' -f1
    exit
fi

if [ "${cmd}" == "lib" -a ${jobs} -gt 0 ]; then
    jobs=1
fi

#
# Source the configuration files
#
source config.in
source ${config_file}

case `echo ${SIMD} | awk '{ print tolower($0)}'` in
    novec)
	# set to zero: remove the specific vector version
	SIMD_size=0
	;;
    sse)
	SIMD_size=16
	;;
    avx)
	SIMD_size=32
	;;
    mic)
	SIMD_size=64
	;;
    *)
	echo "Error: SIMD register type \"${SIMD}\" doesn't exist!"
	echo "Run ./generate -h for help."
	exit
	;;
esac

dims_tiny=`echo ${dims_tiny} | tr "\n" " "` # remove \n
dims_small=`echo ${dims_small} | tr "\n" " "` # remove \n

#
# Dump configuration
#
echo "Config file                   : \"${config_file}\""
echo "Number of tasks per each node : ${ntasks}"
echo "Number of batch jobs          : ${jobs}"
echo "WLM                           : ${wlm}"
echo "SIMD register type            : ${SIMD}"
echo "Limit time batch              : ${wtime}"
echo "COMMAND                       : \"${cmd}\""
echo

#
# Set working variables
#
config_file_name=`basename ${config_file}`
out_dir="${base_out_dir}_${config_file_name}"
OMP_NUM_THREADS=1

if [ ${jobs} -gt 0 ]; then
    source ${def_config_dir}/${wlm}.wlm

    case $cmd in
	tiny1|small1|lib|check1)
	    run_cmd=batch_cmd
	    ;;
	tiny2|small2|check2)
	    run_cmd=true
	    ;;
    esac
else
    # Require 1 job
    jobs=1
fi

type_label="_"
case "${data_type}" in
    1 )
	type_label+="d"
	gemm="DGEMM"
        strdat="REAL(KIND=KIND(0.0D0))"
	;;
    2 )
	type_label+="s"
	gemm="SGEMM"
	strdat="REAL(KIND=KIND(0.0))"
	;;
    3 )
	type_label+="z"
	gemm="ZGEMM"
	strdat="COMPLEX(KIND=KIND(0.0D0))"
	;;
    4 )
	type_label+="c"
	gemm="CGEMM"
	strdat="COMPLEX(KIND=KIND(0.0))"
	;;
esac
case "${transpose_flavor}" in
    1 )
	type_label+="nn"
	ta="N"
	tb="N"
	decl="A(M,K), B(K,N)"
	lds="LDA=M ; LDB=K"
	;;
    2 )
	type_label+="tn"
	ta="T"
	tb="N"
	decl="A(K,M), B(K,N)"
	lds="LDA=K ; LDB=K"
	;; 
    3 )
	type_label+="nt"
	ta="N"
	tb="T"
	decl="A(M,K), B(N,K)"
	lds="LDA=M ; LDB=N"
	;;
    4 )
	type_label+="tt"
	ta="T"
	tb="T"
	decl="A(K,M), B(N,K)"
	lds="LDA=K ; LDB=N"
	;;
esac

run_dir="run"
make_file="Makefile."
case $cmd in
    tiny1|tiny2)
	run_dir+="_tiny"
	make_file+="tiny"
	;;
    small1|small2)
	run_dir+="_small"
	make_file+="small"
	;;
    lib|check1|check2)
	run_dir+="_lib"
	make_file+="lib"
	;;
esac
run_dir+="${type_label}"
make_file+="${type_label}_${config_file_name}"

tiny_file="tiny_gen_optimal${type_label}_${config_file_name}.out"
small_file="small_gen_optimal${type_label}_${config_file_name}.out"
test_file="test_smm${type_label}_${config_file_name}"

library="smm${type_label}_${config_file_name}"
archive="../lib/lib${library}.a"

#
# Run the command!
#

mkdir -p ${run_dir}

case $cmd in
    tiny1|tiny2)
	do_generate_tiny
	if [ "$cmd" == "tiny1" -a -n "${run_cmd}" ]; then
	    echo
	    echo "Wait for completion of all jobs, then run with tiny2 command for collecting all results."
	    echo
	fi
	;;
    small1|small2)
	do_generate_small
	if [ "$cmd" == "small1" -a -n "${run_cmd}" ]; then
	    echo
	    echo "Wait for completion of all jobs, then run with small2 command for collecting all results."
	    echo
	fi
	;;
    lib)
	do_generate_lib
	echo
	echo "Remember to run the check* commands to check the correctness and performance of the library."
	echo
	;;
    check1|check2)
	do_check
	if [ "$cmd" == "check1" -a -n "${run_cmd}" ]; then
	    echo
	    echo "Wait for completion of all jobs, then run with check2 command for collecting all results."
	    echo
	fi
	;;
esac

