#!/usr/bin/perl -w
# /Trilinos/commonTools/test/utilities/runtests

################################################################################
# The Trilinos Project - runtests
# 
# Mike Phenow, Jim Willenbring
#
# This is a utility for running Trilinos packages tests.  It assumes you have
# an existing build directory containing properly configured and built 
# packages and tests.  It also requires that packages have test definition 
# files in thier test directory.
#
################################################################################

use strict;

# Variable Declarations ========================================================

# Command line arguments:

my $trilinosDir;        # Trilinos directory        (required argument) 
my $comm;               # comm                      (required argument) 
my $buildDir;           # build directory           (required argument) 
my $category;           # test category             (required argument) 
my $maxProc;            # maximum processors        (default: 4)
my $mpiStart;           # mpi startup command       (default: NONE)
my $mpiPing;            # mpi ping command          (default: NONE)
my $mpiStop;            # mpi shutdown command      (default: NONE)
my $mpiGo;              # mpi-go command            (default: mpirun -np )
my @packages;           # array of packages to test
my $shortCircuit;       # quit with a non-zero exit code as soon as anything fails

my $outputDir;          # output directory          (default: .)
my $simpleDirName;      # leave off timestamp for sake of database reporting
my $coverage;           # compute coverage data

my $verbosity;          # verbosity level           (default: 1)
my $logVerbosity;       # log file verbosity level  (default: 0)

my @testListAoH;        # test list array of hash
                        #   'package'       =>  package
                        #   'definitionDir' =>  package/test/
                        #   'directory'     =>  vector/
                        #   'descendDirs'   =>  YES
                        #   'type'          =>  MPI/
                        #   'name'          =>  test.exe
                        #   'command'       =>  mpi-go -np16 ./vector/test.exe myVector
                        #   'exitStatus'    =>  [compareTestOutput,STRING-COMPARISON,RETURN-CODE]
                        #   'passString'    =>  my test passed string
                        #   'commandFile'   =>  my_command_file
                        #   'masterFile'    =>  my_master_file

my $resultsDir;         # absolute path to results directory

# Constants
my $v0 = "0";           # quiet
my $v1 = "1";           # normal verbosity
my $v2 = "2";           # level 2 verbosity
my $v3 = "4";           # level 3 verbosity
        
################################################################################
# Execution ####################################################################
################################################################################

getArgs();
init();
run();
cleanUp();

################################################################################
# Subroutines ##################################################################
################################################################################

    ############################################################################
    # getArgs()
    #
    # Parse command line arguments.
    #
    #   - args:     NONE
    #
    #   - returns:  NONE
    #

    sub getArgs {
        
        # Argument variables that don't need to be global.
        my $quiet;
        my $help;
        my $packageList;

        # Gather command line arguments.
        use Getopt::Long;
        GetOptions( "trilinos-dir=s" => \$trilinosDir,
                    "comm=s" => \$comm,
                    "build-dir=s" => \$buildDir,
                    "category=s" => \$category,
                    "max-proc=i" => \$maxProc,
                    "mpi-start=s" => \$mpiStart,
                    "mpi-ping=s" => \$mpiPing,
                    "mpi-stop=s" => \$mpiStop,
                    "mpi-go=s" => \$mpiGo,
                    "packages=s" => \$packageList,
                    "short-circuit" => \$shortCircuit,
                    "output-dir=s" => \$outputDir,
                    "simple-dir-name" => \$simpleDirName,
                    "coverage" => \$coverage,
                    "verbosity=i" => \$verbosity,
                    "log-verbosity=i" => \$logVerbosity,
                    "quiet" => \$quiet,
                    "help" => \$help );

        # Print help and exit.
        if ($help) { 
            printHelp();
            exit;
        }
        
        # Enforce and/or prepare arguments.
        
        # check for existance of trilinos-dir argument and actual directory.
        if (!$trilinosDir) {
            die "trilinos-dir value required, see --help for more information\n"; 
        } else {
            if (!stat($trilinosDir)) {
                die "cannot stat trilinos-dir: $trilinosDir\n";
            }
        }
        
        # Check for existance and acceptable value of comm.
        if (!$comm) {
            die "comm value required, see --help for more information\n";
        } else {
            if (!($comm eq "serial" || $comm eq "mpi")) {
                die "comm value must be \"serial\" or \"mpi\"\n";
            }
        }
        
        # Check for existance of build-dir argument and actual directory.
        # If it isn't given as an absolute path, prepend the Trilinos path.
        if (!$buildDir) {
            die "build-dir value required, see --help for more information\n"; 
        } else {
            if ($buildDir !~ m/^\//) {
                $buildDir = "$trilinosDir/$buildDir";
            }
            if (!stat($buildDir)) {
                die "cannot stat build-dir: $buildDir\n";
            }
        }
        
        # Check for existance and acceptable value of category argument.
        # This may be changed from a 'die' to a 'warn' in the future if
        # developers decide they want to define their own, non-standard groups.
        if (!$category) {
            die "category value required, see --help for more information\n"; 
        } 
        
        # Check for existance of max-proc.
        if (!$maxProc) {
            $maxProc = 4;   # reasonable guess?
        } else {
            if ($maxProc < 1) {
                die "max-proc value must be at least 1\n";
            }
        }
        
        # Check for valid mpi-start command.
        if ($mpiStart) { 
            $mpiStart =~ m/^(\S+)/;
            my $mpiStartProg = $1;
            `which $mpiStartProg`;
            if ($?) {
                die "cannot find mpi-start command: $mpiStartProg\n";
            }
        } else {
            $mpiStart = ""; 
        }
        
        # Check for valid mpi-ping command.
        if ($mpiPing) { 
            $mpiPing =~ m/^(\S+)/;
            my $mpiPingProg = $1;
            `which $mpiPingProg`;
            if ($?) {
                die "cannot find mpi-ping command: $mpiPingProg\n";
            }
        } else {
            $mpiPing = ""; 
        }
        
        # Check for valid mpi-stop command.
        if ($mpiStop) {
            $mpiStop =~ m/^(\S+)/;
            my $mpiStopProg = $1;
            `which $mpiStopProg`;
            if ($?) {
                die "cannot find mpi-stop command: $mpiStopProg\n";
            }
        } else {
            $mpiStop = ""; 
        }
        
        # Check for valid mpi-go command and set mpi-go environment variable.
        if (!$mpiGo) { 
            $mpiGo = "mpirun -np "; 
        }
        if ($comm eq "mpi") {
            $mpiGo =~ m/^(\S+)/;
            my $mpiGoProg = $1;
            `which $mpiGoProg`;
            if ($?) {
                die "cannot find mpi-go command: $mpiGoProg\n";
            } else {
                $ENV{'TRILINOS_TEST_HARNESS_MPIGO_COMMAND'} = $mpiGo;
            }
        }
        
        # Split up comma-separated list of packages
        if ($packageList) { @packages = split(',',$packageList); }
        
        # Check for existance of output directory, use it to create the
        # complete path for the results directory, and create the results
        # directory.
        if (!$outputDir) {
            $outputDir = "";
            $resultsDir = "$trilinosDir/commonTools/test/utilities/results"; 
        } else {
            if ($outputDir =~ m/^\//) {
                $resultsDir = $outputDir;
            } else {
                $resultsDir = "$trilinosDir/commonTools/test/utilities/$outputDir";
            }
        }
        if (!stat($resultsDir)) {
            system("mkdir $resultsDir") == 0 or die "cannot create $resultsDir, died";
        }
        
        # Set verbosity level to corresponding constant.  0, 1, 2, and 3 are
        # used for the levels the user may specify, but they are stored as 0,
        # 1, 2, and 4 so they can be combined and processed with bitwise
        # operations.
        if ($verbosity) {
            if      ($verbosity == 0) { $verbosity = $v0; }
            elsif   ($verbosity == 1) { $verbosity = $v1; }
            elsif   ($verbosity == 2) { $verbosity = $v2; }
            elsif   ($verbosity == 3) { $verbosity = $v3; }
        } else {
            $verbosity = $v1; 
        }
        
        # Set log verbosity level to corresponding constant.  Numbering scheme
        # is the same for the log verbosity as it is for the standard verbosity.
        # There is not distinct output and log output, the if a log verbosity
        # is given, then that level of output is written to a file instead of
        # to standard out.  
        if ($logVerbosity) {
            if      ($logVerbosity == 0) { $logVerbosity = $v0; }
            elsif   ($logVerbosity == 1) { $logVerbosity = $v1; }
            elsif   ($logVerbosity == 2) { $logVerbosity = $v2; }
            elsif   ($logVerbosity == 3) { $logVerbosity = $v3; }        
        } else {
            $logVerbosity = $v1; 
        }
        
        # Set quiet mode--same as passing --verbosity=0.
        if ($quiet) {
            $verbosity = $v0; 
        }            
        
    } # getArgs()

    ############################################################################
    # init()
    #
    # Prepares varibles.
    #
    #   - args:     NONE
    #
    #   - returns:  NONE
    #

    sub init {
           
        # Capture and format start time.
        (my $se, my $mn, my $hr, my $da, my $mo, my $yr) = (localtime)[0..5];
        $yr = sprintf("%02d", $yr % 100);
        $mo = sprintf("%02d", $mo+1);
        $da = sprintf("%02d", $da);
        $hr = sprintf("%02d", $hr);
        $mn = sprintf("%02d", $mn);
        $se = sprintf("%02d", $se);
        my $startTime = $yr."-".$mo."-".$da."_".$hr.".".$mn.".".$se;
        
        # Create results directory.
        if (!$simpleDirName) {
            $resultsDir .= "/$startTime-test";
        } else {
            $resultsDir .= "/test";
        }
        system("mkdir $resultsDir") == 0 or die "cannot create $resultsDir, died";
        
        # Print list of variables for debugging.
        my $message = "";
        $message .= "init():\n";
        $message .= "  \$trilinosDir = $trilinosDir\n";
        $message .= "  \$comm = $comm\n";
        $message .= "  \$buildDir = $buildDir\n";
        $message .= "  \$category = $category\n";
        $message .= "  \$maxProc = $maxProc\n";
        $message .= "  \$mpiStart = $mpiStart\n";
        $message .= "  \$mpiPing = $mpiPing\n";
        $message .= "  \$mpiStop = $mpiStop\n";
        $message .= "  \$mpiGo = $mpiGo\n";
        $message .= "  \$outputDir = $outputDir\n";
        $message .= "  \$verbosity = $verbosity\n";
        $message .= "  \$logVerbosity = $logVerbosity\n";
        $message .= "  \$startTime = $startTime\n";
        $message .= "  \$resultsDir = $resultsDir\n";
        $message .= "  \n";
        printMessage($message, $v3);
        
    } # init()
    
    ############################################################################
    # run()
    #
    # Moves into build-dir/packages, looks for files and runs tests based on
    # @testListAoH compiled by compileTestList.
    #
    #   - args:     NONE
    #
    #   - returns:  NONE
    #

    sub run {
        
        # cd to packages directory in build directory.
        chdir $trilinosDir or die "cannot chdir to $trilinosDir, died";
        chdir "packages/" or die "cannot chdir to packages/, died";
        
        # Compile list of test definition files (using File::Find instead
        # of `find` to improve portability).
        use File::Find;
        find(\&compileTestList, ".");
                
        # Print @testListAoH
        my $message = "";
        $message .= "run():\n";
        $message .= "  \@testListAoH:\n";
        for my $i (0 .. $#testListAoH) {
            $message .= "    test $i:\n";
            foreach my $key (keys %{ $testListAoH[$i] }) { 
                $message .= "      $key: ".$testListAoH[$i]{$key}."\n";                
            }
        }
        $message .= "\n";
        printMessage($message, $v3);
                                                
        # Grab general information values for inclusion in machine info file. 
        my $hostName = "";          my $dnsName = "";           my $ipAddress = "";     
        my $operatingSystem = "";   my $kernelName = "";        my $kernelRelease = "";
        my $kernelVersion = "";     my $processor = "";         my $machineHardware = "";
        my $hardwarePlatform = "";  my $badCmd = 0;          
        $badCmd = system('hostname -s > /dev/null 2>&1');
        if (!$badCmd) {
            chomp($hostName=`hostname -s`);
        } else {
            $hostName=getOptionalOutput("uname -n");
        }
        $dnsName=getOptionalOutput("hostname -d");
        $ipAddress=getOptionalOutput("hostname -i");
        $operatingSystem=getOptionalOutput("uname -o");
        $kernelName=getOptionalOutput("uname -s");
        $kernelRelease=getOptionalOutput("uname -r");
        $kernelVersion=getOptionalOutput("uname -v");
        $processor=getOptionalOutput("uname -p");
        $machineHardware=getOptionalOutput("uname -m");
        $hardwarePlatform=getOptionalOutput("uname -i");
        # Grab the repository branch tag
        my $branchTag = "";
        my $homeDirContents = `ls $trilinosDir`;
        if ($homeDirContents =~ m/CVS/) {
            my $cvsDirContents = `ls $trilinosDir/CVS`;
            if ($cvsDirContents =~ m/Tag/) {
                $branchTag = `cat $trilinosDir/CVS/Tag`;
                $branchTag =~ s/^T//;
            } else {
                $branchTag = "development";
            }      
        } else {
            $branchTag = "unknown";
        }    
                
        # Use global @testListAoH array of hash to run tests and report results
        
        my $testsTotal = 0;
        my $testsPassed = 0;
        my $testsFailed = 0;
        
        # Start MPI
        if ($comm eq "mpi" && $mpiStart) { system("$mpiStart 1>/dev/null 2>/dev/null"); }
        
        # For each test entry...
        for my $i (0 .. $#testListAoH) {
        
            # cd to packages directory in build directory.
            chdir $buildDir or die "cannot chdir to $buildDir, died";
            chdir "packages/" or die "cannot chdir to packages/, died";
            chdir $testListAoH[$i]{'definitionDir'} or next; # Potentially revist... 
            if ($testListAoH[$i]{'descendDirs'} eq "YES") {                
                chdir $testListAoH[$i]{'directory'} or die "cannot chdir to ".$testListAoH[$i]{'directory'}.", died";
            }
            
            # If valid mpi-ping command, check to see if mpi is running and,
            # if not, restart it.
            if ($comm eq "mpi" && $mpiPing && system("$mpiPing 1>/dev/null 2>/dev/null")) {
                if ($mpiStart) { system("$mpiStart 1>/dev/null 2>/dev/null"); }
            }
                
            # Print progress message.
            $message = $testListAoH[$i]{'package'}." - ".$testListAoH[$i]{'name'}."...";
            $message = sprintf("%-55s", $message);
            printMessage($message, $v1+$v2+$v3);
        
            # Capture and format test start time.
            (my $se, my $mn, my $hr, my $da, my $mo, my $yr) = (localtime)[0..5];
            $yr = sprintf("%02d", $yr % 100);
            $mo = sprintf("%02d", $mo+1);
            $da = sprintf("%02d", $da);
            $hr = sprintf("%02d", $hr);
            $mn = sprintf("%02d", $mn);
            $se = sprintf("%02d", $se);
            my $testStartTime = $yr."-".$mo."-".$da." ".$hr.":".$mn.":".$se;
            my $testStartTimeForFilename = $yr."-".$mo."-".$da."_".$hr.".".$mn.".".$se;
            my $startSeconds = time();
        
            # Run test
            my $testOutput = "";
            $testOutput = `$testListAoH[$i]{'command'} 2>&1`;
            my $testExitStatus = $?;
        
            # Trim $testOutput.
            $testOutput = substr ($testOutput, -1000000);
            
            # If defined to do so, run compareTestOutput to get the correct
            # exit status.
            if ($testListAoH[$i]{'exitStatus'} eq "compareTestOutput") {
                
                my $testOutputFile = $testListAoH[$i]{'directory'}."/".$testListAoH[$i]{'name'}.".out.tmp";
                open (TEMP, ">$testOutputFile") 
                    or die "can't open test result file $testOutputFile for writing, died";
                print TEMP $testOutput;
                close TEMP;
                
                my $ctoCommand = "python $trilinosDir/commonTools/test/utilities/compareTestOutput ";
                $ctoCommand .= $testListAoH[$i]{'commandFile'}." ";
                $ctoCommand .= $testListAoH[$i]{'masterFile'}." ";
                $ctoCommand .= "$testOutputFile 2>&1";
                my $ctoOutput = "";
                $ctoOutput = `$ctoCommand`;
                $testExitStatus = $?;
                $testOutput .= "\n\n### compareTestOutput Output ###\n\n";
                $testOutput .= $ctoOutput;
                
                system("rm -f $testOutputFile > /dev/null 2>&1");
            } 
            
            # Do simple string comparison.
            elsif ($testListAoH[$i]{'exitStatus'} eq "STRING-COMPARISON") {
                if ($testListAoH[$i]{'passString'}) { 
                    my $passString = $testListAoH[$i]{'passString'};   
                    if ($testOutput =~ m/$passString/i) { 
                        $testExitStatus = 0; 
                    } else {
                        $testExitStatus = 86; 
                    }
                } else {                
                    if ($testOutput =~ m/End Result:\s*TEST PASSED/i) { 
                        $testExitStatus = 0; 
                    } else {
                        $testExitStatus = 86; 
                    }
                }
            }
            
            # EXIT-STATUS is neither compareTestOutput or STRING-COMPARISON,
            # so use the actual exit status...unless it is 0 and there is
            # obvious fatal output.
            else {
                if (!$testExitStatus) { 
                
                    #  On some platforms, mpirun returns 0 when run on 
                    #  non-binary files, such as scripts.  The following 
                    #  hack tests for this and changes the exit status from 
                    #  0 to 13.
                    if ($testOutput =~ m/process did not invoke MPI_INIT/) {
                        $testExitStatus = 13;
                    }
                    
                }
            }
            
            if (!defined $testOutput) { 
                print "\n\n";
                print "package: ".$testListAoH[$i]{'package'}."\n";
                print "definitionDir: ".$testListAoH[$i]{'definitionDir'}."\n";
                print "directory: ".$testListAoH[$i]{'directory'}."\n";
                print "type: ".$testListAoH[$i]{'type'}."\n";
                print "name: ".$testListAoH[$i]{'name'}."\n";
                print "command: ".$testListAoH[$i]{'command'}."\n";
                print "\n";                
            }
            
            # Tally result
            $testsTotal++;
            if ($testExitStatus) { $testsFailed++; }
            else { $testsPassed++; }
        
            # Capture and format test stop time.
            ($se, $mn, $hr, $da, $mo, $yr) = (localtime)[0..5];
            $yr = sprintf("%02d", $yr % 100);
            $mo = sprintf("%02d", $mo+1);
            $da = sprintf("%02d", $da);
            $hr = sprintf("%02d", $hr);
            $mn = sprintf("%02d", $mn);
            $se = sprintf("%02d", $se);
            my $testStopTime = $yr."-".$mo."-".$da." ".$hr.":".$mn.":".$se;
            my $stopSeconds = time();            
            my $runSeconds = $stopSeconds - $startSeconds;
                            
            # Print pass/fail message.
            $message = ($testExitStatus==0?"  passed ":"! FAILED ");
            printMessage($message, $v1+$v2+$v3);
            $message = ($runSeconds==0?"<1":$runSeconds);
            $message = sprintf("%6s", $message);
            printMessage($message, $v1+$v2+$v3);
            $message = " second".($runSeconds>1?"s":"")."\n";
            printMessage($message, $v1+$v2+$v3);
            
            # Create results file.
            my $resultsFile = "$resultsDir/";
            $resultsFile .= $testListAoH[$i]{'package'}."_";
            $resultsFile .= $testListAoH[$i]{'name'}."_";
            
            my $testNumber = "00";
            while (stat "$resultsFile"."$testNumber.txt") {
                $testNumber++;
            }           
            $resultsFile .= $testNumber.".txt";
            open (RESULTS, ">$resultsFile") 
                or die "can't open test result file $resultsFile for writing, died";
                
            my $resultsString = "";
            
            $resultsString .= "HOST_NAME            = $hostName\n";
            $resultsString .= "DNS_NAME             = $dnsName\n";
            $resultsString .= "IP_ADDRESS           = $ipAddress\n";
            $resultsString .= "OPERATING_SYSTEM     = $operatingSystem\n";
            $resultsString .= "KERNEL_NAME          = $kernelName\n";
            $resultsString .= "KERNEL_RELEASE       = $kernelRelease\n";
            $resultsString .= "KERNEL_VERSION       = $kernelVersion\n";
            $resultsString .= "PROCESSOR            = $processor\n";
            $resultsString .= "MACHINE_HARDWARE     = $machineHardware\n";
            $resultsString .= "HARDWARE_PLATFORM    = $hardwarePlatform\n";
            $resultsString .= "\n";
            $resultsString .= "TRILINOS_DIR         = $trilinosDir\n";
            $resultsString .= "BRANCH_TAG           = $branchTag\n";
            $resultsString .= "\n";
            $resultsString .= "BUILD_DIR            = $buildDir\n";
            $resultsString .= "COMM                 = $comm\n";
            $resultsString .= "\n";
            $resultsString .= "PACKAGE_TEST_DIR     = ".$testListAoH[$i]{'definitionDir'}."\n";
            $resultsString .= "PACKAGE              = ".lc($testListAoH[$i]{'package'})."\n";
            $resultsString .= "\n";
            $resultsString .= "TEST_CATEGORY        = $category\n";
            $resultsString .= "\n";
            $resultsString .= "TEST_DIR             = ".$testListAoH[$i]{'directory'}."\n";
            $resultsString .= "TEST_NAME            = ".$testListAoH[$i]{'name'}."\n";
            $resultsString .= "TYPE                 = ".$testListAoH[$i]{'type'}."\n";
            $resultsString .= "COMMAND              = ".$testListAoH[$i]{'command'}."\n";
            $resultsString .= "\n";
            $resultsString .= "TEST_NUMBER          = $testNumber\n";
            $resultsString .= "START_TIME           = $testStartTime\n";
            $resultsString .= "STOP_TIME            = $testStopTime\n";
            $resultsString .= "RUN_TIME             = ".($runSeconds==0?"<1":$runSeconds)." seconds\n";
            $resultsString .= "\n";
            $resultsString .= "EXIT_STATUS          = $testExitStatus\n";
            $resultsString .= "RESULT               = ".($testExitStatus==0?"pass":"fail")."\n";
            $resultsString .= "\n";
            $resultsString .= "TEST_OUTPUT {{{\n\n$testOutput\n\n}}}\n";
            
            print RESULTS $resultsString;
            
            close RESULTS;
            
            if ($shortCircuit && $testExitStatus) {
                print "short-circuit mode:  test failed:  quitting.\n";
                return 77;
            }
            
        } # foreach @testListAoH
        
        # Print test summary.
        $message = "\n";
        $message .= "  Tests Passed: ".sprintf("%3s", "$testsPassed")."\n";
        $message .= "  Tests Failed: ".sprintf("%3s", "$testsFailed")."\n";
        $message .= " -------------------\n";
        $message .= "  Tests Total:  ".sprintf("%3s", "$testsTotal")."\n\n";
        printMessage($message, $v1+$v2+$v3);
        
        # Stop MPI
        if ($comm eq "mpi" && $mpiStop) { system("$mpiStop 1>/dev/null 2>/dev/null"); }
        
        # Perform coverage tasks.
        if ($coverage) { coverage(); }
        
    } # run()
    
    ############################################################################
    # compileTestList()
    #
    # If File::Find::dir is a test directory and File::Find::name is a 
    # definition file, parse it, and fill global @testListAoH.  Note that we are 
    # automatically 'chdir'-ed to File::Find::dir.
    #
    #   - args:     NONE
    #
    #   - returns:  NONE
    #

    sub compileTestList {
    
        # Is this file in a 'test' directory?
        # Is this file named 'definition'?
        if ($File::Find::dir =~ m/test$/ && $File::Find::name =~ m/definition$/) {

            # Figure out package name for reporting's sake.
            (my $package = $File::Find::dir) =~ m/\.\/((external\/)?(.*?)\/test)/;
            $package = $3;
	    my $packageTestPath = $1;
            
            # If the user supplies a list of packages to test (to the exclusion
            # of all others), return if this package isn't on that list.
            if ($#packages >= 0) {
                my $found = 0;
                foreach my $testPackage (@packages) {
                    if ($package =~ m/^$testPackage$/i) { $found = 1; }
                }
                if (!$found) { return; }
            }
            
            # Parse definition file into array of hashes of arrays.
            my @definitionAoHoA;
            (my $failures, my @result) = parseDefinition();
            
            if (!$failures) {
                @definitionAoHoA = @result;            
            } else {
                my $message = "";
                $message .= "! $failures error".($failures>1?"s":"")." parsing ";
                $message .= "$package definition file:\n";
                $message .= $result[0];
                printMessage($message, $v1+$v2+$v3);
                return;
            }
            
            # For each test group...
            for my $i (0 .. $#definitionAoHoA) {
            
                # Should we skip this group?
                # At this point, this decision should be made purely on the
                # meaning contained within the properties, not on well-formedness.
                # Well-formedness decisions ought to have been made in
                # parseDefinition() and parseTestGroup().
                my $skipThisGroup = 0;
            
                # Is this OS excluded?
                if (!$skipThisGroup) {
                    my $os = getOptionalOutput("uname -o");
                    foreach my $x_os (@{ $definitionAoHoA[$i]{'X-OS'} }) {
                        if ($x_os =~ m/$os/) { $skipThisGroup = 1; }
                    } 
                }
            
                # Is this architecture excluded?
                if (!$skipThisGroup) {
                    my $arch = getOptionalOutput("uname -m");
                    foreach my $x_arch (@{ $definitionAoHoA[$i]{'X-ARCH'} }) {
                        if ($x_arch =~ m/$arch/) { $skipThisGroup = 1; }
                    }
                } 
            
                # Is this host excluded?
                my $host = "";  # we need this a little later to coerce atlantis to run mpi tests
                if (!$skipThisGroup) {
                    $host = getOptionalOutput("uname -n");
                    foreach my $x_host (@{ $definitionAoHoA[$i]{'X-HOST'} }) {
                        if ($x_host =~ m/$host/) { $skipThisGroup = 1; }
                    }
                } 
            
                # Does this group define tests for this comm?
                if (!$skipThisGroup) {
                    my $foundMatch = 0;
                    foreach my $item (@{ $definitionAoHoA[$i]{'COMM'} }) {
                        if ($item =~ m/$comm/i) { $foundMatch = 1; }
                    }
                    if (!$foundMatch) { $skipThisGroup = 1; }
                }
            
                # Does this group apply to this category?
                if (!$skipThisGroup) {
                    my $foundMatch = 0;
                    foreach my $item (@{ $definitionAoHoA[$i]{'CATEGORIES'} }) {
                        if ($item =~ m/$category/i) { $foundMatch = 1; }
                    }
                    if (!$foundMatch) { $skipThisGroup = 1; }
                }
                
                # This OS, architecture, and host are not excluded from this
                # test group and this test group defines tests for this comm.
                # Using TESTS, DIRS, ARGS, and COMM, add a completed hash entry
                # to the global @testListAoH array of hashes for every test.
                if (!$skipThisGroup) {
                    
                    # If no args are specified, we need a blank one to get into
                    # the foreach loop and avoid using an uninitialized value
                    # concatenating the arg to the command string.
                    if (!defined @{ $definitionAoHoA[$i]{'ARGS'} }) {
                    
                        # If ARG-MASTER-PAIRS is being used, generate arguments.
                        if (@{ $definitionAoHoA[$i]{'ARG-MASTER-PAIRS'} }[0]) {
                            
                            # We need to keep the master file with the argument
                            # for now, but set it off somehow so as not to get
                            # lost in the shuffle.
                            foreach my $arg (@{ $definitionAoHoA[$i]{'ARG-MASTER-PAIRS'} }) {
                                $arg =~ s/^(.*?):(.*?)$/$1:::$2:::/;
                                push (@{ $definitionAoHoA[$i]{'ARGS'} }, $arg);
                            }
                        } 
                        
                        # Generate empty argument as described above.
                        else {                        
                            ${ @{ $definitionAoHoA[$i]{'ARGS'} } }[0] = ("");
                        }
                    }
                    
                    # Since a user of this test utility can only specify a comm
                    # of 'serial' or 'mpi' and since a package developer can
                    # only specify a custom or non-custom comm in any given test
                    # group, we can determine exactly which of the four is
                    # applicable for this run and this test group.
                    my $isCustom = 0;
                    foreach my $item (@{ $definitionAoHoA[$i]{'COMM'} }) {
                        if ($item =~ m/CUSTOM/i) { $isCustom = 1; }
                    }                
                    
                    # Add a complete hash entry for each executable file found
                    # in the directories listed in DIRS.
                    foreach my $dir (@{ $definitionAoHoA[$i]{'DIRS'} }) {
                    
                        # Open the corresponding, analogous directory in the
                        # build directory.  I'm not convinced this method of
                        # constructing that path is bullet-proof.  It may be
                        # making assumptions about the precise contents of
                        # $package and $dir.
                        my $builtTestDirPath = "$buildDir/packages/$packageTestPath/$dir";
                    
                        if (opendir(DIR, $builtTestDirPath)) {
                            while (defined (my $file = readdir(DIR))) {
                                my $absFile = "$buildDir/packages/$packageTestPath/$dir/$file";
                                if ( isValidTestExec("$buildDir/packages/$packageTestPath/$dir",$file) ) {
                                  push( @{ $definitionAoHoA[$i]{'TESTS'} }, "$dir/$file");
                                }
                            }
                            closedir(DIR);
                        }
                    }
                    
                    # Add a complete hash entry for each test listed in TESTS.
                    foreach my $test (@{ $definitionAoHoA[$i]{'TESTS'} }) {

                        # Check the build directory to make sure that every test
                        # is actually there and executable.  It is done above
                        # for tests specified with "DIRS = ", but this is needed
                        # here to make sure tests specified with "TESTS = " are
                        # actually built too.

                        if (-x "$buildDir/packages/$packageTestPath/$test") {
                        
                            # Each argument specified gets its own test run.
                            foreach my $argOrig (@{ $definitionAoHoA[$i]{'ARGS'} }) {
                            
                                # Because modifying arg for one test appears to
                                # leave it modified for the next test, we need to 
                                # explicitly make a copy of it for each test.  This
                                # is for the sake of the ARG-MASTER-PAIRS
                                # functionality.
                                my $arg = $argOrig;
                            
                                # Prepare the hash entry for this test that will
                                # end up in @testListAoH.
                                my %testEntry = ();                            
                                $testEntry{'package'} = $package;
                                
                                use File::Basename;        
                                my $testDir = dirname($test);
                                my $testName = basename($test);
                                
                                $testEntry{'definitionDir'} = $File::Find::dir;
                                $testEntry{'directory'} = $testDir;
                                $testEntry{'name'} = $testName;
                                
                                # Should we descend to the level of the test
                                # before invoking it?  (default is no)
                                my $descendDirs = ${ \@{ $definitionAoHoA[$i]{'DESCEND_DIRS'} } }[0];
                                if (defined $descendDirs && $descendDirs =~ m/YES/i) {
                                    $testEntry{'descendDirs'} = "YES";
                                } else {
                                    $testEntry{'descendDirs'} = "NO";
                                }
                        
                                # Add values for compareTestOutput.
                                if (@{ $definitionAoHoA[$i]{'EXIT-STATUS'} }[0] && 
                                    @{ $definitionAoHoA[$i]{'EXIT-STATUS'} }[0] =~ m/compareTestOutput/i) {
                                    $testEntry{'exitStatus'} = "compareTestOutput";
                                    
                                    # COMMAND-FILE
                                    if (@{ $definitionAoHoA[$i]{'COMMAND-FILE'} }[0]) {
                                        $testEntry{'commandFile'} = "$testDir/".@{ $definitionAoHoA[$i]{'COMMAND-FILE'} }[0];
                                    } else {
                                        $testEntry{'commandFile'} = "";
                                    }
                                    
                                    # MASTER-FILE...
                                    if (@{ $definitionAoHoA[$i]{'MASTER-FILE'} }[0]) {
                                        $testEntry{'masterFile'} = "$testDir/".@{ $definitionAoHoA[$i]{'MASTER-FILE'} }[0];
                                    }                                 
                                    # or ARG-MASTER-PAIR?
                                    elsif (@{ $definitionAoHoA[$i]{'ARG-MASTER-PAIRS'} }[0]) {
                                        my $masterFile = $arg;
                                        $masterFile =~ s/^.*?:::(.*?):::$/$1/;
                                        $testEntry{'masterFile'} = "$testDir/$masterFile";
                                        $arg =~ s/:::.*?::://;
                                        if (-f "$buildDir/packages/$packageTestPath/$testDir/$arg") {
                                            $arg = "./$testDir/$arg";
                                        }
                                        
                                    } else {
                                        $testEntry{'masterFile'} = "";
                                    } 
                                } 
                                
                                # Add values for STRING-COMPARISON
                                elsif (@{ $definitionAoHoA[$i]{'EXIT-STATUS'} }[0] && 
                                       @{ $definitionAoHoA[$i]{'EXIT-STATUS'} }[0] =~ m/STRING-COMPARISON/i) {
                                    $testEntry{'exitStatus'} = "STRING-COMPARISON";
                                    if (@{ $definitionAoHoA[$i]{'EXIT-STATUS'} }[0] =~ m/STRING-COMPARISON\((.*)\)/i) {
                                        $testEntry{'passString'} = $1;                                
                                    }
                                } 
                                
                                # EXIT-STATUS is neither compareTestOutput or 
                                # STRING-COMPARISON, so it defaults to RETURN-CODE.
                                else {
                                    $testEntry{'exitStatus'} = "RETURN-CODE";                            
                                }
                            
                                # I'm not sure if there really needs to be a
                                # CUSTOM-SERIAL value for comm.  I don't know how
                                # it would be treated differently.  For now,
                                # CUSTOM-SERIAL will be treated the same as SERIAL.
                                # For that matter, it appears that CUSTOM-MPI is
                                # the same as SERIAL and CUSTOM-SERIAL.
                                if ($comm eq "serial") {
                                    if ($isCustom) {
                                        $testEntry{'type'} = "CUSTOM-SERIAL";
                                        if ($testEntry{'descendDirs'} eq "YES") {
                                            $testEntry{'command'} = "./".$testEntry{'name'}." ".$arg;
                                        } else { 
                                            $testEntry{'command'} = "./".$testEntry{'directory'}."/".$testEntry{'name'}." ".$arg;
                                        }
                                        push(@testListAoH, { %testEntry });
                                    } else {
                                        $testEntry{'type'} = "SERIAL";
                                        if ($testEntry{'descendDirs'} eq "YES") {
                                            $testEntry{'command'} = "./".$testEntry{'name'}." ".$arg;
                                        } else { 
                                            $testEntry{'command'} = "./".$testEntry{'directory'}."/".$testEntry{'name'}." ".$arg;
                                        }
                                        push(@testListAoH, { %testEntry });
                                    }
                                    
                                } elsif ($comm eq "mpi") {
                                    if ($isCustom) {
                                        $testEntry{'type'} = "CUSTOM-MPI";
                                        if ($testEntry{'descendDirs'} eq "YES") {
                                            $testEntry{'command'} = "./".$testEntry{'name'}." ".$arg;
                                        } else { 
                                            $testEntry{'command'} = "./".$testEntry{'directory'}."/".$testEntry{'name'}." ".$arg;
                                        }
                                        push(@testListAoH, { %testEntry });
                                    } else {
                                        $testEntry{'type'} = "MPI";
                                        
                                        # Make decisions about the number of 
                                        # prcessors to run on.
                                        
                                        my $mpiString = "";
                                        foreach my $arg (@{ $definitionAoHoA[$i]{'COMM'} }) {
                                            if ($arg =~ m/MPI/i) { $mpiString = $arg; }
                                        }
                                        
                                        # optional () present
                                        if ($mpiString =~ m/MPI\((.*?\))/i) { 
                            
                                            my $rangeString = $1;
                                            my @rangeList = $rangeString =~ m/(.*?)[ )]/g;
                                            
                                            foreach my $range (@rangeList) {
                                            
                                                my $rangeMin = 1;
                                                my $rangeMax = 1;
                                                my $numProc = 1;
                                                
                                                if ($range =~ m/-/) {
                                                    $range =~ m/(\d*)-(\S*)/;
                                                    $rangeMin = $1;
                                                    $rangeMax = $2;
                                                    if ($rangeMax eq "*") { $rangeMax = $maxProc; }
                                                    if ($rangeMin <= $maxProc) { 
                                                        if ($rangeMax <= $maxProc) { 
                                                            $numProc = $rangeMax;
                                                        } elsif ($rangeMax > $maxProc) {
                                                            $numProc = $maxProc;
                                                        }
                                                        if ($host =~ m/atlantis/) {                                                            
                                                            if ($testEntry{'descendDirs'} eq "YES") {
                                                                $testEntry{'command'} = "$mpiGo"."$numProc $buildDir/packages/$packageTestPath/".$testEntry{'directory'}."/".$testEntry{'name'}." ".$arg;
                                                            } else { 
                                                                $testEntry{'command'} = "$mpiGo"."$numProc $buildDir/packages/$packageTestPath/".$testEntry{'directory'}."/".$testEntry{'name'}." ".$arg;
                                                            }
                                                        } else {
                                                            if ($testEntry{'descendDirs'} eq "YES") {
                                                                $testEntry{'command'} = "$mpiGo"."$numProc ./".$testEntry{'name'}." ".$arg;
                                                            } else { 
                                                                $testEntry{'command'} = "$mpiGo"."$numProc ./".$testEntry{'directory'}."/".$testEntry{'name'}." ".$arg;
                                                            }
                                                        }
                                                        push(@testListAoH, { %testEntry });
                                                    }
                                                } else {
                                                    if ($range eq "*") { $range = $maxProc; }
                                                    $numProc = $range;
                                                    if ($numProc <= $maxProc) { 
                                                        if ($host =~ m/atlantis/) {
                                                            if ($testEntry{'descendDirs'} eq "YES") {
                                                                $testEntry{'command'} = "$mpiGo"."$numProc $buildDir/packages/$packageTestPath/".$testEntry{'directory'}."/".$testEntry{'name'}." ".$arg;
                                                            } else { 
                                                                $testEntry{'command'} = "$mpiGo"."$numProc $buildDir/packages/$packageTestPath/".$testEntry{'directory'}."/".$testEntry{'name'}." ".$arg;
                                                            }
                                                        } else {
                                                            if ($testEntry{'descendDirs'} eq "YES") {
                                                                $testEntry{'command'} = "$mpiGo"."$numProc ./".$testEntry{'name'}." ".$arg;
                                                            } else { 
                                                                $testEntry{'command'} = "$mpiGo"."$numProc ./".$testEntry{'directory'}."/".$testEntry{'name'}." ".$arg;
                                                            }
                                                        }
                                                        push(@testListAoH, { %testEntry });
                                                    }
                                                }                                        
                                            }
                                            
                                        # no optional () present
                                        } else { 
                                            
                                            my $numProc = $maxProc;
                                            if ($host =~ m/atlantis/) {
                                                if ($testEntry{'descendDirs'} eq "YES") {
                                                    $testEntry{'command'} = "$mpiGo"."$numProc $buildDir/packages/$packageTestPath/".$testEntry{'directory'}."/".$testEntry{'name'}." ".$arg;
                                                } else { 
                                                    $testEntry{'command'} = "$mpiGo"."$numProc $buildDir/packages/$packageTestPath/".$testEntry{'directory'}."/".$testEntry{'name'}." ".$arg;
                                                }
                                            } else {
                                                if ($testEntry{'descendDirs'} eq "YES") {
                                                    $testEntry{'command'} = "$mpiGo"."$numProc ./".$testEntry{'name'}." ".$arg;
                                                } else { 
                                                    $testEntry{'command'} = "$mpiGo"."$numProc ./".$testEntry{'directory'}."/".$testEntry{'name'}." ".$arg;
                                                }
                                            }
                                            push(@testListAoH, { %testEntry });
                                        
                                        }
                                        
                                    } # non-CUSTOM MPI
                                    
                                } # $comm eq MPI
                                                    
                            } # foreach $arg
                            
                        } # if -x $test
                        
                    } # foreach $test
                
                } # if !$skipThisGroup
                
            } # foreach test group
            
        } # if 'test/' and 'definition'
        
    } # compileTestList()
    
    ############################################################################
    # parseDefinition()
    #
    # Assuming we have been placed in a 'test/' directory containing a 
    # 'definition' file, break that file into raw (.*){.*} string for parsing
    # by parseTestGroup().  No global decision-making takes place about whether 
    # or not to run any tests defined here.
    #
    #   - args:     NONE
    #
    #   - returns:  $definitionAoHoA    (array of test groups, which contains...
    #                                    hashes of property names, which contain...
    #                                    arrays of property values.)
    #

    sub parseDefinition {
        my $status = 0;
        my $errorString = "";
    
        # Array of hashs of arrays that will be returned when all is said and done.
        my @definitionAoHoA = ();
        
        # Open definition file for reading.
        if (!open (DEFINITION, "<definition")) {
            $status++;
            $errorString .= "! Can't open definition file.\n";
        }

        # Read definition file into one string.
        undef $/;               # undefine input record separator
        my $file=<DEFINITION>;  # copy entire file
        $/ = "\n";              # restore it to default newline
        close DEFINITION;
        
        # Clean up file.
        $file =~ s/#.*$//mg;        # remove everything after any #
        $file =~ s/\n\s*\n/\n/g;    # remove any blank lines
        $file =~ s/^\s*//mg;        # remove leading spaces
        $file =~ s/\s*$//mg;        # remove trailing spaces
        $file =~ s/\n//g;           # remove all newlines (force ourselves
                                    # to avoid making assumptions about formatting)
        
        # Escape key characters
        $file =~ s/\\\{/\[\[\[ESCAPED_LEFT_BRACE\]\]\]/g;
        $file =~ s/\\\}/\[\[\[ESCAPED_RIGHT_BRACE\]\]\]/g;
        $file =~ s/\\\,/\[\[\[ESCAPED_COMMA\]\]\]/g;
        $file =~ s/\\\;/\[\[\[ESCAPED_SEMICOLON\]\]\]/g;
        
        # Split definition file into groups.
        # This gives us an array of strings of the essential form (.*){.*}.
        my @rawTestGroups = $file =~ m/(\(.*?\)\s*\{.*?\})/g;
        
        # Step through the list of raw testGroups, parsing each with
        # parseTestGroup() and adding to the array.
        foreach my $rawTestGroup (@rawTestGroups) {
            (my $failures, my %result) = parseTestGroup($rawTestGroup);
            if (!$failures) {
                push(@definitionAoHoA, { %result });
            } else {
                $status += $failures;
                $errorString .= $result{"errorString"};
            }
        }
        
        # print @definitionAoHoA
        my $message = "";
        $message .= "parseDefinition():\n";
        $message .= "  \@definitionAoHoA:\n";
        for my $i (0 .. $#definitionAoHoA) {
            $message .= "    test group $i:\n";
            foreach my $key (keys %{ $definitionAoHoA[$i] }) { 
                $message .= "      $key\n";
                foreach my $item (@{ $definitionAoHoA[$i]{$key} }) {
                    $message .= "        '$item'\n";
                }
            }
        }
        $message .= "\n";
        printMessage($message, $v3);
        
        if (!$status) {         # Return properly parsed definition file.
            return ($status, @definitionAoHoA);
        } else {                # Return $errorString string.
            return ($status, ($errorString));
        }
    
    } # parseDefinition()
    
    ############################################################################
    # parseTestGroup()
    #
    # Given a raw (.*){.*} string, parse it into a hash of arrays.
    #
    #   - args:     $rawGroupString     (raw (.*){.*} string)
    #
    #   - returns:  $status             (success(0)/failure(!0) status)
    #               %testGroupHoA       (hash of property names, which contains...
    #                                    arrays of property values.)
    #               $errorString        (record of malformed group error messages)
    #

    sub parseTestGroup {
        my $rawGroupString = $_[0];
        my $status = 0;
        my $errorString = "";
    
        # Hash of arrays that will be returned when all is said and done.
        my %testGroupHoA = ();
        
        # Break raw string into category (.*) and property {.*} sections
        my $categoryString = "";
        if ($rawGroupString =~ m/\((.+?\))/) {
            $categoryString = $1;
        } else {
            $status++;
            $errorString .= "! Malformed test group:  bad category list.\n";
        }
        
        my $propertyListString = "";
        if ($rawGroupString =~ m/\{(.+?)\}/) {
            $propertyListString = $1;
        } else {
            $status++;
            $errorString .= "! Malformed test group:  bad property list.\n";
        }
        
        # Category section (.*):       
        my @categories = $categoryString =~ m/\s*(\w+?)\s*[,)]\s*/g;
        $testGroupHoA{'CATEGORIES'} = [ @categories ];
        
        # Property section {.*}:       
        # Grap each raw 'PROPERTY = VALUE-1, VALUE-N;' string.
        my @rawPropertyStrings = $propertyListString =~ m/(.+?;)/g;        
        foreach my $propertyString (@rawPropertyStrings) {
            
            # Grab property name.
            my $key = "";
            if ($propertyString =~ m/\s*((\w|-)+?)\s*=.*?;/) {
                $key = $1;
            } else {
                $status++;
                $errorString .= "! Malformed test group:  bad property list.\n";
            }
            
            # Grab the raw values list string.
            my $valuesString = "";
            if ($propertyString =~ m/\s*\w+?\s*=\s*(.*?;)/) {
                $valuesString = $1;
                # Replace $ABS_TOP_SRCDIR with $trilinosDir
                $valuesString =~ s/\$ABS_TOP_SRCDIR/$trilinosDir/g;
            } else {
                $status++;
                $errorString .= "! Malformed test group:  bad property list.\n";
            }
            
            # Break raw values list string into array of values.
            # Don't break it into a list if it's the EXIT-STATUS value--doing
            # so would break up a test-passed string appended to 
            # STRING-COMPARISON.
            my @valueList = ();
            if ($key eq "EXIT-STATUS") {
                push(@valueList, $valuesString);
            } else {
                @valueList = $valuesString =~ m/\s*(.+?)\s*[,;]/g;                
                # replace [[[ESCAPED_PUNCTUATION]]] with actual punctuation
                foreach my $escapedString (@valueList) {
                    $escapedString =~ s/\[\[\[ESCAPED_LEFT_BRACE\]\]\]/\{/g;
                    $escapedString =~ s/\[\[\[ESCAPED_RIGHT_BRACE\]\]\]/\}/g;
                    $escapedString =~ s/\[\[\[ESCAPED_COMMA\]\]\]/\,/g;
                    $escapedString =~ s/\[\[\[ESCAPED_SEMICOLON\]\]\]/\;/g;       
                }                
            }
            
            # Add new key=>(values) entry in array.
            $testGroupHoA{$key} = [ @valueList ];
        }
        
        # Validate test group:
        
        # Enforce existance of COMM property.
        if (!$testGroupHoA{'COMM'}) {
            $status++;            
            $errorString .= "! Malformed test group:  no COMM property.\n";
        }
        
        # Enforce existance of TESTS or DIRS properties.
        if (!($testGroupHoA{'TESTS'} || $testGroupHoA{'DIRS'})) {
            $status++;
            $errorString .= "! Malformed test group:  no TESTS or DIRS properties.\n";
        }
        
        # Enforce existance of COMMAND-FILE if EXIT-STATUS is compareTestOutput
        if ($testGroupHoA{'EXIT-STATUS'} && $testGroupHoA{'EXIT-STATUS'}[0] =~ m/compareTestOutput/i) {
            if (!$testGroupHoA{'COMMAND-FILE'}) {
                $status++;
                $errorString .= "! Malformed test group:  EXIT-STATUS=compareTestOutput, but no COMMAND-FILE property.\n";
            }
            if ($testGroupHoA{'ARG-MASTER-PAIRS'}) {
                if ($testGroupHoA{'ARGS'} || $testGroupHoA{'MASTER-FILE'}) {
                    $status++;
                    $errorString .= "! Malformed test group:  if ARG-MASTER-PAIRS is present, neither ARGS or MASTER can be.\n";
                }
            }
        }
        
        # Enforce either all 'custom' or no 'custom'.
        my $foundCustom = 0;
        my $foundNonCustom = 0;
        foreach my $item (@{$testGroupHoA{'COMM'}}) {
            if ($item =~ m/CUSTOM/i) {
                $foundCustom = 1;
            } else {
                $foundNonCustom = 1;
            }
        }
        if ($foundCustom && $foundNonCustom) {
            $status++;
            $errorString .= "! Malformed test group:  cannot have both CUSTOM and non-CUSTOM\n";
            $errorString .= "  comms listed in the same test group.\n";        
        }
        
        # Add more validations as necessary to catch errors sooner...
        
        if (!$status) {         # Return properly parsed test group.
            return ($status, %testGroupHoA);
        } else {                # Return $errorString string.
            return ($status, ("errorString", $errorString));
        }
    
    } # parseTestGroup()
    
    ############################################################################
    # coverage()
    #
    # Perform coverage tasks.
    #
    #   - args:     NONE
    #
    #   - returns:  NONE
    #

    sub coverage {
    
        chdir $trilinosDir or die "cannot chdir to $trilinosDir, died";
        
        my $command = "./commonTools/test/coverage/gcovmerge --with-directories ";
        $command .= "--exclude=test/ --exclude=example/ --exclude=examples/ ";
        $command .= "--exclude=doc/ --source-dir=$trilinosDir/packages --with-count ";
        $command .= "--with-percent --with-full-paths $buildDir/packages";
        my $output = `$command`;
        my $exitStatus = $?;
        
        chdir $resultsDir or die "cannot chdir to $resultsDir, died";
        mkdir "coverage" or die "cannot create directory \"coverage\", died";
        
        my $summaryFile = "$resultsDir/coverage.txt";
        open (SUMMARY, ">$summaryFile") 
            or die "can't open test coverage summary file $summaryFile for writing, died";
              
        print SUMMARY $output;        
        close SUMMARY;
        
        use File::Find;
        find (\&createTree, ("$buildDir/packages"));
        find (\&copyGcov, ("$buildDir/packages"));
        
    } # coverage()
    
    ############################################################################
    # createTree()
    #
    # Create directory tree under test/coverage
    #
    #   - args:     NONE
    #
    #   - returns:  NONE
    #

    sub createTree {
    
        use File::Basename;
        my $absFile = $File::Find::name;
        my $file = basename($absFile);
        my $dir = dirname($absFile);
        
        # Is this file a directory?
        if (-e $file && -d $file && $file ne "packages" && $file ne "CVS") {
        
            my $destination = $absFile;
            $destination =~ s/$buildDir\/packages/$resultsDir\/coverage/;
            
            if (!(-e $destination && -d $destination)) {
                mkdir "$destination" or die "cannot create directory $destination, died";   
            }
        
        }
           
    } # createTree()
    
    ############################################################################
    # copyGcov()
    #
    # Copy all .gcov files
    #
    #   - args:     NONE
    #
    #   - returns:  NONE
    #

    sub copyGcov {
    
        use File::Basename;
        my $absFile = $File::Find::name;
        my $file = basename($absFile);
        my $dir = dirname($absFile);
        
        if ($file =~ m/\.gcov$/) {
            
            my $destination = $dir;
            $destination =~ s/$buildDir\/packages/$resultsDir\/coverage/;
            
            my $output = `cp $absFile $destination`;
            my $exitStatus = $?;
        
        }
           
    } # copyGcov()
    
    ############################################################################
    # cleanUp()
    #
    # Clean up environment variables, temp files, etc.
    #
    #   - args:     NONE
    #
    #   - returns:  NONE
    #

    sub cleanUp {
    
        delete $ENV{'TRILINOS_TEST_HARNESS_MPIGO_COMMAND'}; 
           
    } # cleanUp()
    
    ############################################################################
    # printMessage()
    #
    # Prints an event if the verbosity is set.
    #
    #   - args:     $message        (message to be printed)
    #               $level          (verbosity level of message)
    #
    #   - returns:  NONE
    #

    sub printMessage {
        my $message = $_[0];
        my $level = $_[1];
        
        if ($verbosity & $level) {
            print $message;
        }
        
        if ($logVerbosity & $level) {
            my $log = $resultsDir."/log.txt";
            open (LOG, ">>$log")
                or die "can't open $log";
            print LOG $message;
            close LOG;
        }
    } # printMessage()

    ############################################################################
    # printHelp()
    #
    # Prints help output.
    #
    #   - args:     NONE
    #
    #   - returns:  NONE
    #

    sub printHelp {
        print "runtests - The Trilinos Test Utility\n";
        print "\n";
        print "Usage:  perl runtests --trilinos-dir=/home/user/Trilinos --comm=mpi --build-dir=MPI\n";
        print "        --category=all\n";
        print "\n";
        print "Options:\n";
        print "\n";
        print "  --trilinos-dir=DIR     Specify the absolute path to the top-level Trilinos\n";
        print "                         directory that contains this program.\n";
        print "                         Example: /home/user/Trilinos\n";
        print "                         REQUIRED.\n";
        print "\n";
        print "  --comm=COMM            Specify the type of build, either \"serial\" or \"mpi\"\n";
        print "                         REQUIRED.\n";
        print "\n";
        print "  --build-dir=DIR        Specify the path of the build directory that contains\n";
        print "                         the tests you would like run.  If a relative path is\n";
        print "                         given, it is assumed that it is in the given Trilinos\n";
        print "                         directory.\n";
        print "                         REQUIRED.\n";
        print "\n";
        print "  --category=CATEGORY    Specify the category of tests to be run.  This must be\n";
        print "                         one of the predefined tests types:  FRAMEWORK,\n";
        print "                         CHECKIN, PERFORMANCE, SCALABILITY, or ALL.  See\n";
        print "                         README-definition for more information.\n";
        print "                         REQUIRED.\n";
        print "\n";
        print "  --max-proc=N           Specify the maximum number of processors on which to\n";
        print "                         run a test.\n";
        print "                         Default: 4\n";
        print "\n";
        print "  --mpi-start=COMMAND    Specify the mpi startup command for this system.\n";
        print "                         Default: NONE\n";
        print "\n";
        print "  --mpi-ping=COMMAND     Specify the mpi 'ping' or 'status' command for this system.\n";
        print "                         Default: NONE\n";
        print "\n";
        print "  --mpi-stop=COMMAND     Specify the mpi shutdown command for this system.\n";
        print "                         Default: NONE\n";
        print "\n";
        print "  --mpi-go=COMMAND       Specify the mpi-go command for this system.\n";
        print "                         Default: \"mpirun -np \"\n";
        print "\n";
        print "  --packages=P1[,P2]     Specify the packages to test.  Separate multiple\n";
        print "                         packages with commas.  Omit to test all built\n";
        print "                         packages with tests defined.\n";
        print "\n";
        print "  --short-circuit        Quit with a non-zero exit code as soon as\n";
        print "                         anything fails.\n";
        print "\n";
        print "  --output-dir=DIR       Specify the directory in which to create the directory\n";
        print "                         containing the test results.\n";
        print "                         Default: \".\"\n";
        print "\n";
        print "  --verbosity=LEVEL      0 = no non-fatal ouput (same as --quiet)\n";
        print "                         1 = normal output (default)\n";
        print "                         2 = level 2 verbosity\n";
        print "                         3 = level 3 verbosity\n";
        print "\n";
        print "  --log-verbosity=LEVEL  0 = no log\n";
        print "                         1 = normal output (default)\n";
        print "                         2 = level 2 verbosity\n";
        print "                         3 = level 3 verbosity\n";
        print "\n";
        print "  --quiet                Produce no non-fatal output.\n";
        print "\n";
        print "  --help                 Print this help output and exit.\n";
        print "\n";
        print "Notes:\n";
        print "  - For more information, see README-runtests in\n";
        print "    Trilinos/commonTools/test/utilities/\n";
        print "    or visit http://software.sandia.gov/trilinos/developer/\n";
        print "\n";
    } # printHelp()

    ############################################################################
    # getOptionalOutput()
    #
    # Runs a system command and gets its output if it succeeds then returns
    # its output.  The program suppress output to stderr of the program does
    # not run correctly.
    #
    #   - args:     the shell command
    #
    #   - returns:  the optional output string
    #

    sub getOptionalOutput {
      my $cmnd_in = shift;
      my $cmnd = "$cmnd_in > /dev/null 2>&1";
      my $returnVal = system($cmnd);
      my $outputStr="";
      chomp($outputStr = `$cmnd_in`) if($returnVal == 0);
      return $outputStr;
    } # getOptionalOutput()


    ############################################################################
    # isValidTestExec()
    #
    # Determines if the input file is a valid executable file that can
    # be a test program.
    #
    #   - args:
    #               the absolute path to a candidate file
    #               the file name itself
    #
    #   - returns:  True if the file is a valid executable test program
    #

    sub isValidTestExec {
      my $abs_path = shift;
      my $file = shift;
      my $absFile = "$abs_path/$file";
      if(

         (-x $absFile) &&        # Must be executable!

         (!-d $absFile) &&       # Must not be a directory!

         ($file !~ m/^\./) &&    # We do not run hidden files!
         ($file !~ m/\.obj$/) &&  # These files are made exectuable by the
         ($file !~ m/\.ilk$/) &&  # Windows Intel C++ compiler and should not
         ($file !~ m/\.pdb$/)     # be run as test programs!

        )
      {
        return 1;
      }
      return 0;
    } # isValidTestExec()
