#!/bin/bash

###################################################################################
# do_regtest some info [Joost VandeVondele, 2005-02-25]
#
# Regression testing cp2k ... 
#    - maintain code quality
#    - helps rapid developemnent and refactoring
#
# What does a regtest do
#    - automatically update cp2k with the current CVS version (including new tests)
#    - performs a realclean build of the source
#    - executes a list of tests
#    - compares the results (outputs) with those of the last known result (reference)
#    - produces a summary
#
# How to set up a regtest
#    - you must be able to build and run cp2k on the given machine, the rest should be 'easy'
#    - decide on a directory for doing the regtest, there will be plenty of files in this dir
#      (after a while) so make it something like $HOME/rt
#    - checkout a version of cp2k into $HOME/rt. 
#      If you set up your cvs account so that you don't need to provide
#      a passwd for doing cvs update, things will be more convenient later on.
#    - set up the arch files so that you can cleanly build cp2k (test this)
#    - cp $HOME/rt/cp2k/tools/do_regtest $HOME/rt
#    - modify the do_regtest script to match your local environment (e.g. specify 
#      if you wish a sdbg or a sopt build, which compiler, gmake, ....) see below
#    - execute './do_regtest' regularly (i.e. after comitting new code)
#
# Interpretation of the results
#  test can be:
#    - 'OK' if the results match those of a previous run precisely. The execution time is also given.
#    - 'NEW' if they have not been executed previously. The reference result is generated 
#      automatically in this run. Tests can also be 'NEW' if they have been reset, i.e. been newly 
#      added to the TEST_FILES_RESET files.
#    - 'RUNTIME FAILURE' if they stopped unexpectedly (e.g. core dump, or stop)
#    - 'WRONG RESULT' if they produce a result that deviates (even a tiny bit) from an old reference
#  the last two options generally mean that a bug has been introduced, which requires investigation.
#  since regtesting only yields information relative to a previously known result, it is most useful
#  to do a regtest before and after you make changes.
#
# Adding/resetting/creating tests to the testsuite
#  these is fully controlled by the following files in the cp2k/tests directories
#  -TEST_DIRS  : is just a list of directories that contain tests. You can add your directory here.
#  -TEST_FILES : the list of input files that need to be executed. You can add your file name here.
#                adding a comment about what it tests might help later debugging problems if a regtest
#                fails
#  -TEST_FILES_RESET : you can add files for which the reference output became invalid (e.g. bug fix)
#                      to this list fo files. However be absolutely sure that the change is due to
#                      a bug fix, do not reset these that fail because of unclear reasons. Try to add
#                      a comment to the cvs message and/or the file itself
#  -TEST_TYPES : this file allows you to create a new test type. I.e. to specify for which words should
#                be grepped and what field should be used in the numerical comparison. 
#
# Command line switches to the do_regtest script (also configurable from within the script)
#  -nocvs   : do not access the CVS for any updating, makes regtesting fully local
#  -quick   : rebuild the code if needed, but do not perform a realclean before (noquick is not needed anymore)
#  -noreset : do not reset the reference outputs automatically
#  -cvsdate string : specify any string to cvs update (most likely used as "-D 2005-02-17")
#  -skipdir string : this switch can repeat, exclude certain dirs from regtesting, useful to 
#                    speed-up regtesting after very localised changes (e.g. -skipdir QS/regtest)
#  -restrictdir string : this switch can repeat, restrict regtesting to certain dirs, useful to 
#                        speed-up regtesting after very localised changes (e.g. -restrictdir QS/regtest)
#  -config string : loads a site/compiler/environment specific configuration
#
# Script configuration. The value of the follow variables can be redefined, see below
#   dir_base, FORT_C_NAME, cp2k_version, dir_triplet, cp2k_dir, cp2k_prefix, cp2k_postfix,
#   make, awk, datum_full, datum_short,
#   nocvs, quick, cvsdate, noreset, ndirtoskip, skip_dirs, restrict_dir, ndirtorestrict
#
########################################################################################################
#
# THESE VARIABLES WILL NEED CHANGING FOR YOUR LOCAL CONFIGURATION
#
# - dir_base: the base directory for testing (e.g. $HOME/rt)
# - FORT_C_NAME: compiler selection (e.g. intel)
# - cp2k_version: sopt,sdbg,popt....
# - dir_triplet: the result of tools/get_arch_name, where the executable can be found 
# - cp2k_dir: normally cp2k
# - maxtasks: how many instances of cp2k should run simultaneously (~> #CPUs)
# - emptycheck: useful for automatic testers, no testing if nothing changed in CVS (YES|NO)
# - leakcheck: if using g95, set this variable to "YES" in order to get memory leak checking
#
# The following variable definitions can now be loaded from a
# site-specific configuration using the -config option. Create one
# configuration file for each architecture/compiler using these as a
# template (and don't forget to include a modified cp2k_prefix for MPI
# runs as well!).
export LC_ALL=C
export FORT_C_NAME=gfortran
dir_base=/data/vjoost/clean/cp2k
cp2k_version=pdbg
dir_triplet=Linux-x86-64-gfortran
cp2k_dir=cp2k
maxtasks=16
emptycheck="NO"
leakcheck="NO"

#
# The following variables typically need no changes on Linux machine, but might need changes on
# other an OS
#

# *** how to execute an input file [ cp2k_prefix input cp2k_postfix ]
# Leave empty for serial, uncomment for parallel
cp2k_run_prefix="mpiexec -np 2"
#cp2k_run_prefix=""
cp2k_run_postfix=
#cp2k_prefix="( ulimit -t 300; ${dir_base}/${cp2k_dir}/exe/${dir_triplet}/cp2k.${cp2k_version}"
#cp2k_postfix=" )"
#cp2k_prefix="poe ${dir_base}/${cp2k_dir}/exe/${dir_triplet}/cp2k.${cp2k_version}"
#cp2k_postfix="-llfile ${dir_base}/llfile"

# *** make and awk
make=make
#make=gmake
awk=awk
#awk=nawk

# *** a short and long version of the data, in a format that CVS understands
datum_full=`date --iso-8601="seconds"`
datum_short=`date --iso-8601="seconds"`
#datum_full=`date '+%Y-%m-%dT%H:%M:%S+0100'`
#datum_short=`date '+%Y-%m-%d'`

# *** default settings for command line switches
nocvs="nocvs"
quick="quick"
cvsdate="-D $datum_full"
noreset="reset"
ndirstoskip=0
skip_dirs[1]=""
ndirstorestrict=0
restrict_dirs[1]=""

###################################################################################
#
# From here on no changes to the script should be needed
#
###################################################################################
#
# command line argument passing
#
###################################################################################
while [ $# -ge 1 ]; do
case $1 in
  # build a list of directories to skip
  -skipdir) let ndirstoskip=ndirstoskip+1;
        skip_dirs[ndirstoskip]=$2;
        shift;
        ;;
  # build a list of directories to restrict, i.e. only matching dirs will be run 
  -restrictdir) let ndirstorestrict=ndirstorestrict+1;
        restrict_dirs[ndirstorestrict]=$2;
        shift;
        ;;
  # load system-specific configuration
  -config)
        source $2 || { echo "Error loading configuration from $2" ; exit 1 ; }
        shift
        ;;
  # do not update the CVS
  -nocvs) nocvs="nocvs";;
  # do not reset reference outputs
  -noreset) noreset="noreset";;
  # do not do a realclean before building
  -quick) quick="quick";;
  # specify the full string "-D 2005-02-01" to get a check-out of a specific date
  -cvsdate) cvsdate=$2; shift;;
  *)  break;;
esac
shift
done

if [ "${ARCH}" ]; then
    ARCH_SPEC="ARCH=${ARCH}"
fi

# *** how to execute an input file [ cp2k_prefix input cp2k_postfix ]
# Unless cp2k_prefix is overridden in the sourced -config file, the
# default is used.
cp2k_prefix=${cp2k_prefix:-"${cp2k_run_prefix} ${dir_base}/${cp2k_dir}/exe/${dir_triplet}/cp2k.${cp2k_version}"}
cp2k_postfix=${cp2k_postfix:-"${cp2k_run_postfix}"}

# allow the config file to set the maximum allowed time. Useful for valgrinded runs
job_max_time=${job_max_time:-"300"}

###################################################################################
#
# set up the initial directory structures
#
###################################################################################
test_types_file=${dir_base}/${cp2k_dir}/tests/TEST_TYPES
dir_last=${dir_base}/LAST-${dir_triplet}-${cp2k_version}
dir_out=${dir_base}/TEST-${dir_triplet}-${cp2k_version}-${datum_short}
changelog_diff=${dir_out}/ChangeLog.diff
changelog_diff_tests=${dir_out}/ChangeLog-tests.diff
error_description_file=${dir_out}/error_summary
mkdir -p ${dir_out}
mkdir -p ${dir_last}
rm -fR ${error_description_file}
touch  ${error_description_file}
touch  ${dir_last}/ChangeLog-tests
if [[ ${leakcheck} == "YES" ]]; then
   memory_description_file=${dir_out}/memory_summary
   memory_summary=${dir_base}/memory_leaks.txt
   rm -fR ${memory_description_file}
   touch  ${memory_description_file}
   echo "" > ${memory_summary}
fi

###################################################################################
#
# simple function to end the tests all in the same way
#
###################################################################################
function end_test() {
echo "--------------------------------------------------------------------------"
date
echo "*************************** testing ended ********************************"
exit $1
}

###################################################################################
#
# function to grep for changes in the output. Takes five arguments
#
###################################################################################
function do_test_grep(){
 output_new=$1
 output_old=$2
 error_file=$3
 grep_string=$4
 grep_field=$5 
 e1=`grep -a "${grep_string}" ${output_old} | tail -1 | ${awk} -v f=${grep_field} '{print $f}'`
 e2=`grep -a "${grep_string}" ${output_new} | tail -1 | ${awk} -v f=${grep_field} '{print $f}'`
 big=`echo "${e1} ${e2}" | ${awk} '{if($2==0){v=sqrt(($1-$2)^2)}else{v=sqrt((($1-$2)/$2)^2)}; if (v>1.0E-14) printf("%16.8e",v); else printf("0") ;}'`
 case ${big} in
 0)
  # ok, same energy
  return 0 ;;
 *)
  # nope too large
  echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >> ${error_file}
  echo "${output_new} : " >> ${error_file}
  echo " ${grep_string} : old = ${e1} new = ${e2} " >> ${error_file}
  echo " relative error : ${big}  " >> ${error_file}
  echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >> ${error_file}
  return 1 ;;
 esac 
}

###################################################################################
#
# function to select which test to run
#
###################################################################################
function do_test() {
 which_test=$1
 output_new=$2
 output_old=$3
 error_file=$4
 case ${which_test} in
 0) 
   #just be happy you executed
   return 0;;
 *)
   do_test_grep ${output_new} ${output_old} ${error_file} "${test_grep[which_test]}" "${test_col[which_test]}"
   return $? ;;
 esac
}

# *** start testing
echo "*************************** testing started ******************************"
echo " started on " `date`
echo " checking version ${cvsdate} "
echo "--------------------------- CVS ------------------------------------------"

if [[ ${nocvs} != "nocvs" ]]; then
  # *** cvs update src
  cd ${dir_base}/${cp2k_dir}/src
  cvs update -d -A ${cvsdate} &> out
  if (( $? )); then
  echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >> ${error_description_file}
  tail -20 out >> ${error_description_file}
  echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >> ${error_description_file}
  echo "error happened : no cvs update ... bailing out"
  end_test 1
  fi
  echo "cvs src update went fine"
  cp2k_lines=`wc *.F | tail -1 |  ${awk} '{print $1}'`
  echo "cp2k is now ${cp2k_lines} lines .F"
  # *** looks like cvs2cl always generates the changelog for the latest version of the repo, not the current one?
  ${dir_base}/${cp2k_dir}/tools/cvs2cl.pl &> out
  diff ChangeLog ${dir_last}/ChangeLog > ${changelog_diff}
  echo "------- differences --------" >> ${changelog_diff}
  cvs -n update ${cvsdate} &> out
  cat out | grep -v -i "changelog" | grep -v '?' | grep -v 'cvs server: Updating' >> ${changelog_diff} 
  echo "---  changelog diff src  ---"
  cat ${changelog_diff} 
  echo "----------------------------"
  cp ChangeLog ${dir_last}/ChangeLog
  
  # *** cvs update tests
  cd ${dir_base}/${cp2k_dir}/tests
  cvs update -d -A ${cvsdate} &> out
  if (( $? )); then
  echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >> ${error_description_file}
  tail -20 out >> ${error_description_file}
  echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >> ${error_description_file}
  echo "error happened : no cvs update ... bailing out"
  end_test 2
  else
  echo "cvs tests update went fine"
  fi
  ${dir_base}/${cp2k_dir}/tools/cvs2cl.pl &> out
  diff ChangeLog ${dir_last}/ChangeLog-tests > ${changelog_diff_tests}
  echo "------- differences --------" >> ${changelog_diff_tests}
  cvs -n update ${cvsdate} &> out
  cat out | grep -v -i "changelog" | grep -v '?' |  grep -v 'cvs server: Updating' >> ${changelog_diff_tests}
  echo "--- changelog diff tests ---"
  cat ${changelog_diff_tests} 
  echo "----------------------------"
  cp ChangeLog ${dir_last}/ChangeLog-tests
else
  echo "no cvs updating"
fi

# *** check if there is any update or difference that thus requires a rerun
# one day, this requires improvement for speed.
if [[ ${emptycheck} == "YES" ]]; then
   isempty_1=`nl ${changelog_diff} | awk '{print $1}'`
   isempty_2=`nl ${changelog_diff_tests} | awk '{print $1}'`
   
   if [[ ${isempty_1} == "1" && ${isempty_2} == "1" ]]; then
      echo "No changes since last run -- clean exit without testing "
      end_test 100
   else
      echo "Code has changed since last run -- continue regtest "
   fi
fi

# *** make realclean
if [[ ${quick} != "quick" ]]; then
   cd ${dir_base}/${cp2k_dir}/makefiles
   ${make} realclean ${ARCH_SPEC} VERSION=${cp2k_version} &> out
   if (( $? )); then
      echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >> ${error_description_file}
      tail -20 out >> ${error_description_file}
      echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >> ${error_description_file}
      echo "error happened : make realclean VERSION=${cp2k_version} ... bailing out"
      end_test 3
   else
      echo "make realclean VERSION=${cp2k_version} went fine"
   fi
else
  echo "quick testing, no realclean"
fi 

# *** from here failures are likely to be bugs in cp2k
echo "-------------------------compiling cp2k-----------------------------------"
cd ${dir_base}/${cp2k_dir}/makefiles
${make} -j ${maxtasks} ${ARCH_SPEC} VERSION=${cp2k_version} &> out
if (( $? )); then
echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >> ${error_description_file}
tail -40 out >> ${error_description_file}
echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >> ${error_description_file}
echo "error happened : make VERSION=${cp2k_version} "
cat "${error_description_file}"
end_test 4
else
compile_warnings=`grep "Warning:" out | wc | tail -1 |  ${awk} '{print $1}'`
echo "make VERSION=${cp2k_version} went fine (${compile_warnings} warnings)"
fi
echo "-------------------------regtesting cp2k----------------------------------"

###################################################################################
#
# parse the TEST_TYPES file to do different kinds of test (done after cvs update)
#
# tests grep for the last line in the file where a string matches (test_grep) 
# and compares a numeric field at a given column (test_col)
#
# the format of the TEST_TYPES file is (notice the '!' as a field separator, to allow
# for spaces in the test_grep)
#
# Ntest_types
# test_grep_1 ! test_col_1
# test_grep_2 ! test_col_2
# ....
# followed by comment lines
#
###################################################################################
Ntest_types=`awk -v l=1 -v c=1 'BEGIN{FS="!"}{lr=lr+1;if (lr==l) print $c}' ${test_types_file}`
test_grep[0]=""
test_col[0]=1
t=1
while [ $t -le ${Ntest_types} ]; do
test_grep[t]=`${awk} -v l=$t -v c=1 'BEGIN{FS="!"}{lr=lr+1;if (lr==l+1) print $c}' ${test_types_file}`
test_col[t]=`${awk} -v l=$t -v c=2 'BEGIN{FS="!"}{lr=lr+1;if (lr==l+1) print $c}' ${test_types_file}`
let t=t+1
done

###################################################################################
#
# *** now start testing 
# *** for a given directory we do a run on all files in TEST_FILES and
# *** do the test as indicated by the number
# *** files are run in order so that they can e.g. restart
#
###################################################################################
n_runtime_error=0
n_wrong_results=0
n_correct=0
n_tests=0
n_new=0

# Copy the tests into the working regtest directory.
cp -al ${dir_base}/${cp2k_dir}/tests ${dir_out}/tests || \
cp -rpl ${dir_base}/${cp2k_dir}/tests ${dir_out}/tests || ( \
    echo "Could not copy testing directory as hard links."
    echo "Using normal copy instead."
    sleep 10
    cp -rp ${dir_base}/${cp2k_dir}/tests/* ${dir_out}/tests
)

#
# get a list of directories to be tested, taking care of the exclusions
#
dirs=`cat ${dir_out}/tests/TEST_DIRS | grep -v "#"`
newdirs=""
for dir in ${dirs}
do
  match="no"
  t=1
  # *** match to exclusion list
  while [ $t -le ${ndirstoskip} ]; do
     if [[ "${skip_dirs[t]}" == "${dir}" ]]; then
        match="yes" 
     fi
     let t=t+1
  done
  # *** match to the restrict list, if no restrict list is found, all dirs match
  if [ ${ndirstorestrict} -gt 0 ]; then
     restrictmatch="no"
     t=1
     while [ $t -le ${ndirstorestrict} ]; do
        if [[ "${restrict_dirs[t]}" == "${dir}" ]]; then
           restrictmatch="yes" 
        fi
        let t=t+1
     done
  else
    restrictmatch="yes"
  fi

  # *** if not excluded add to list of dirs
  if [[ "${match}" == "no" && "${restrictmatch}" == "yes" ]]; then
     new_dirs="$new_dirs $dir"
  fi
done
dirs=$new_dirs

#
# execute all regtests
#

# Just to be sure, clean possible existing status files.
cd ${dir_out}
mkdir ${dir_out}/status
rm -f ${dir_out}/status/REGTEST_RUNNING-* \
    ${dir_out}/status/REGTEST_TASK_RESULT-* \
    ${dir_out}/status/REGTEST_TASK_TESTS-*


for dir in ${dirs};
do
 #
 # tests in different dirs can run in parallel. We spawn processes up to a given maximum
 #
 task=${dir//\//-}
 (
  touch ${dir_out}/status/REGTEST_RUNNING-$task
  n_runtime_error=0
  n_wrong_results=0
  n_correct=0
  n_tests=0
  n_new=0

  cd ${dir_out}/tests/${dir}
  mkdir -p ${dir_out}/${dir}
  mkdir -p ${dir_last}/${dir}
  touch ${dir_last}/${dir}/TEST_FILES_RESET

  # 
  # first reset reference outputs that have become out-dated since the last run
  #
  if [[ ${noreset} != "noreset" ]]; then
     diff TEST_FILES_RESET ${dir_last}/${dir}/TEST_FILES_RESET > ${dir_out}/${dir}/TEST_FILES_RESET.diff
     cp TEST_FILES_RESET ${dir_last}/${dir}/TEST_FILES_RESET
     nreset=`grep '<' ${dir_out}/${dir}/TEST_FILES_RESET.diff | grep -v '#' |  ${awk} '{c=c+1}END{print c}'`
     for ((itest=1;itest<=nreset;itest++));
     do
        reset_file=`grep '<' ${dir_out}/${dir}/TEST_FILES_RESET.diff | grep -v '#' | ${awk} -v itest=$itest '{c=c+1;if (c==itest) print $2}'`
        rm -f ${dir_last}/${dir}/${reset_file}.out
     done
  fi
  #
  # run the tests now
  #
  echo "Starting tests in ${dir_out}/tests/${dir}"
  echo ">>>>>>>>>>>>>>>>> ${dir_out}/tests/${dir}" > ${dir_out}/status/REGTEST_TASK_TESTS-$task
  ntest=`grep -v "#" TEST_FILES | ${awk} '{c=c+1}END{print c}'`
  t1=`date +%s`
  for ((itest=1;itest<=ntest;itest++));
  do
     n_tests=$((n_tests+1))
     this_test=""
     input_file=`grep -v "#" TEST_FILES | ${awk} -v itest=$itest '{c=c+1;if (c==itest) print $1}'`
     # just one test right now, but this should generalize
     test_types=`grep -v "#" TEST_FILES | ${awk} -v itest=$itest '{c=c+1;if (c==itest) print $2}'`
     output_file=${dir_out}/${dir}/${input_file}.out
     output_last=${dir_last}/${dir}/${input_file}.out
     ( ulimit -t ${job_max_time} -v 2000000 ; ${cp2k_prefix} ${input_file} ${cp2k_postfix} &> ${output_file} )
     # *** cp2k failed obviously
     if (( $? )); then
        echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >> ${error_description_file}
        echo ${output_file} >> ${error_description_file}
        tail -40 ${output_file} >> ${error_description_file}
        echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >> ${error_description_file}
        this_test="RUNTIME FAIL"
        n_runtime_error=$((n_runtime_error+1))
        failed_tests="${failed_tests} ${output_file}"
     else 
        # *** but didn't end !?
        grep -a "ENDED" ${output_file} &> /dev/null
        if (( $? )); then
           echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >> ${error_description_file}
           echo ${output_file} >> ${error_description_file}
           tail -40 ${output_file} >> ${error_description_file}
           echo "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" >> ${error_description_file}
           this_test="RUNTIME FAIL"
           n_runtime_error=$((n_runtime_error+1))
           failed_tests="${failed_tests} ${output_file}"
        else
           # *** still running, you must be joking...
           # *** see if we manage to pass the testing
           # *** but only if we can compare
           if [ -f ${output_last} ]; then
              for test_type in ${test_types};
              do
                 do_test ${test_type} ${output_file} ${output_last} ${error_description_file}
                 if (( $? )); then
                    this_test="WRONG RESULT TEST ${test_type}"
                    n_wrong_results=$((n_wrong_results+1))
                    # *** no further testing
                    break;
                 else
                    n_correct=$((n_correct+1))
                    this_test="OK"
                 fi
              done
           else
              this_test="NEW"
              n_new=$((n_new+1))
           fi
        fi
     fi
     # Keep the output up-to-date
     timing=0
     case ${this_test} in
     "NEW" )
        cp ${output_file} ${output_last} 
        timing=`grep -a "CP2K   " ${output_file} | tail -n 1 | ${awk} '{printf("%6.2f",$NF)}'`
        this_test="${this_test} (${timing} sec)" ;;
     "OK" )
        timing=`grep -a "CP2K   " ${output_file} | tail -n 1 | ${awk} '{printf("%6.2f",$NF)}'`
        this_test="${this_test} (${timing} sec)" ;;
     esac
     if [[ ${leakcheck} == "YES" ]]; then
        dum=`grep -l "Remaining memory" ${output_file}`
        if [[ ${dum} != "" ]]; then
           this_test="${this_test} !"          
           echo "XXXXXXXX  ${output_file} XXXXXXX" >> ${memory_description_file}
           grep -i "Remaining memory"  ${output_file} >> ${memory_description_file}
        fi
     fi 
     printf "%50s %20s\n" "${dir}/${input_file}" "${this_test}" >> ${dir_out}/status/REGTEST_TASK_TESTS-$task
  done
  t2=`date +%s`
  timing_all=`echo "1+($t2)-($t1)" | bc -l`
  printf "%s %8.2f %s\n" "<<<<<<<<<<<<<<<<< ${dir_out}/tests/${dir}" ${timing_all} "sec"  >> ${dir_out}/status/REGTEST_TASK_TESTS-$task
  echo "${n_runtime_error} ${n_wrong_results} ${n_correct} ${n_new} ${n_tests}" > ${dir_out}/status/REGTEST_TASK_RESULT-$task
  cat ${dir_out}/status/REGTEST_TASK_TESTS-$task
  rm -f ${dir_out}/status/REGTEST_TASK_TESTS-$task ${dir_out}/status/REGTEST_RUNNING-$task
 )&

 #
 # here we allow only a given maximum of tasks
 #
 runningtasks=10000
 while (( runningtasks >= maxtasks ))
 do
   # sleep 1
   runningtasks=`ls -1 ${dir_out}/status/REGTEST_RUNNING-* 2> /dev/null | awk 'BEGIN{c=0}{c=c+1}END{print c}'`
 done

done

#
# wait for all tasks to finish
#
wait
#
# generate results
#
for dir in ${dirs};
do
  task=${dir//\//-}
  file=${dir_out}/status/REGTEST_TASK_RESULT-$task
  tmp=`awk '{print $1}' $file`
  n_runtime_error=$((n_runtime_error+tmp))
  tmp=`awk '{print $2}' $file`
  n_wrong_results=$((n_wrong_results+tmp))
  tmp=`awk '{print $3}' $file`
  n_correct=$((n_correct+tmp))
  tmp=`awk '{print $4}' $file`
  n_new=$((n_new+tmp))
  tmp=`awk '{print $5}' $file`
  n_tests=$((n_tests+tmp))
  rm -f $file
done

echo "--------------------------------------------------------------------------"
cat "${error_description_file}"
echo "--------------------------------- summary --------------------------------"
printf "number of FAILED  tests %d\n" ${n_runtime_error}
printf "number of WRONG   tests %d\n" ${n_wrong_results}
printf "number of CORRECT tests %d\n" ${n_correct}
printf "number of NEW     tests %d\n" ${n_new}
printf "number of         tests %d\n" ${n_tests}

if [[ ${leakcheck} == "YES" ]]; then
   echo "--------------------------------------------------------------------------"
   n_leaks=`grep -v "XXXXXXXX" ${memory_description_file} | wc | ${awk} '{print $1}'`
   printf "number of memory  leaks %d\n" ${n_leaks}
   echo "Memory leaks summary" >  ${memory_summary}
   echo "--------------------" >>  ${memory_summary}
   echo "This is the build of $datum_full using g95 (www.g95.org)" >>  ${memory_summary}
   echo "At most 25 leaks are reported per run" >>  ${memory_summary}
   echo "-------------------------------------------------------" >>  ${memory_summary}
   echo "There are ${n_leaks} leaks today " >> ${memory_summary}
   echo "-------------------------------------------------------" >>  ${memory_summary}
   cat  ${memory_description_file} >>  ${memory_summary}
   if [[ -f ${changelog_diff} ]]; then
      echo "----- changelog diff file ------" >>  ${memory_summary}
      cat ${changelog_diff} >> ${memory_summary}
   fi 
   echo "GREPME ${n_runtime_error} ${n_wrong_results} ${n_correct} ${n_new} ${n_tests} ${n_leaks}"
fi

end_test 0
