#!/usr/bin/perl -w
# /Trilinos/commonTools/test/utilities/runmake

################################################################################
# The Trilinos Project - runmake
# 
# Mike Phenow, Jim Willenbring
#
# This is a utility for compiliing Trilinos.  It assumes you have a properly
# configured build tree.  It is of limited value by itself.  It is best used in
# conjunction with runconfigure, as in the utility runbuild.
#
# - Check contents of report.
# - Output package names as they compile successfully.
#
################################################################################

use strict;

# Variable Declarations ========================================================

# Command line arguments:

my $trilinosDir;        # Trilinos directory        (required argument) 
my $buildDir;           # build directory           (required argument) 
my $makeCmd;            # make command              (default: make)
my $noBlame;            # no blame?

my $outputDir;          # output directory          (default: .)
my $verbosity;          # verbosity level           (default: 1)
my $logVerbosity;       # log file verbosity level  (default: 0)
my $startTime;          # start time                (supplied by runbuild)

my $resultsDir;         # absolute path to results directory

my $runStartTime;
my $runStartTimeForFilename;

# Constants
my $v0 = "0";           # quiet
my $v1 = "1";           # normal verbosity
my $v2 = "2";           # level 2 verbosity
my $v3 = "4";           # level 3 verbosity
        
################################################################################
# Execution ####################################################################
################################################################################

getArgs();
init();

my $exitStatus = run();
cleanUp();
exit $exitStatus;

################################################################################
# Subroutines ##################################################################
################################################################################

    ############################################################################
    # getArgs()
    #
    # Parse command line arguments.
    #
    #   - args:     NONE
    #
    #   - returns:  NONE
    #

    sub getArgs {
        
        # Argument variables that don't need to be global.
        my $quiet;
        my $help;

        # Gather command line arguments.
        use Getopt::Long;
        GetOptions( "trilinos-dir=s" => \$trilinosDir,
                    "build-dir=s" => \$buildDir,
                    "make-cmd=s" => \$makeCmd,
                    "no-blame" => \$noBlame,
                    "output-dir=s" => \$outputDir,
                    "verbosity=i" => \$verbosity,
                    "log-verbosity=i" => \$logVerbosity,
                    "start-time=s" => \$startTime,
                    "quiet" => \$quiet,
                    "help" => \$help );
        
        # Print help and exit.
        if ($help) { 
            printHelp();
            exit;
        }
        
        # Enforce and/or prepare arguments.
        
        # check for existance of trilinos-dir argument and actual directory
        if (!$trilinosDir) {
            die "trilinos-dir value required, see --help for more information\n"; 
        } else {
            if (!stat($trilinosDir)) {
                die "cannot stat trilinos-dir: $trilinosDir\n";
            }
        }
        
        # Check for existance of build-dir argument and actual directory.
        # If it isn't given as an absolute path, prepend the Trilinos path.
        if (!$buildDir) {
            die "build-dir value required, see --help for more information\n"; 
        } else {
            if ($buildDir !~ m/^\//) {
                $buildDir = "$trilinosDir/$buildDir";
            }
            if (!stat($buildDir)) {
                die "cannot stat build-dir: $buildDir\n";
            }
        }
        
        # Check for valid make-cmd.
        if ($makeCmd) {
            $makeCmd =~ m/^(\S+)/;
            my $testMakeCommand = $1;
            `which $testMakeCommand`;
            if ($?) {
                die "cannot find make-cmd command: $testMakeCommand\n";
            }
        } else {
            $makeCmd = "make"; 
        }
        
        # Prevent problems with uninitialized value
        if (!$noBlame) {
            $noBlame= "";
        }
        
        # Check for existance of output directory, use it to create the
        # complete path for the results directory, and create the results
        # directory.
        if (!$outputDir) {
            $outputDir = "";
            $resultsDir = "$trilinosDir/commonTools/test/utilities/results"; 
        } else {
            if ($outputDir =~ m/^\//) {
                $resultsDir = $outputDir;
            } else {
                $resultsDir = "$trilinosDir/commonTools/test/utilities/$outputDir";
            }
        }
        if (!stat($resultsDir)) {
            system("mkdir $resultsDir") == 0 or die "cannot create $resultsDir, died";
        }
        
        # Set verbosity level to corresponding constant.  0, 1, 2, and 3 are
        # used for the levels the user may specify, but they are stored as 0,
        # 1, 2, and 4 so they can be combined and processed with bitwise
        # operations.
        if ($verbosity) {
            if      ($verbosity == 0) { $verbosity = $v0; }
            elsif   ($verbosity == 1) { $verbosity = $v1; }
            elsif   ($verbosity == 2) { $verbosity = $v2; }
            elsif   ($verbosity == 3) { $verbosity = $v3; }
        } else {
            $verbosity = $v1; 
        }
        
        # Set log verbosity level to corresponding constant.  Numbering scheme
        # is the same for the log verbosity as it is for the standard verbosity.
        # There is not distinct output and log output, the if a log verbosity
        # is given, then that level of output is written to a file instead of
        # to standard out.  
        if ($logVerbosity) {
            if      ($logVerbosity == 0) { $logVerbosity = $v0; }
            elsif   ($logVerbosity == 1) { $logVerbosity = $v1; }
            elsif   ($logVerbosity == 2) { $logVerbosity = $v2; }
            elsif   ($logVerbosity == 3) { $logVerbosity = $v3; }        
        } else {
            $logVerbosity = $v1; 
        }
        
        # Set quiet mode--same as passing --verbosity=0.
        if ($quiet) {
            $verbosity = $v0; 
        }            
        
    } # getArgs()

    ############################################################################
    # init()
    #
    # Prepares varibles.
    #
    #   - args:     NONE
    #
    #   - returns:  NONE
    #

    sub init {
        
        # We've been passed the --start-time= command line argument by
        # runbuild, so use that instead of capturing our own.
        if ($startTime) {
            $runStartTimeForFilename = $startTime;
            $runStartTime = $runStartTimeForFilename;
            $runStartTime =~ s/_/ /;
            $runStartTime =~ s/\./:/g;
        } else {    
            # Capture and format make start time.
            (my $se, my $mn, my $hr, my $da, my $mo, my $yr) = (localtime)[0..5];
            $yr = sprintf("%02d", $yr % 100);
            $mo = sprintf("%02d", $mo+1);
            $da = sprintf("%02d", $da);
            $hr = sprintf("%02d", $hr);
            $mn = sprintf("%02d", $mn);
            $se = sprintf("%02d", $se);
            $runStartTime = $yr."-".$mo."-".$da." ".$hr.":".$mn.":".$se;
            $runStartTimeForFilename = $yr."-".$mo."-".$da."_".$hr.".".$mn.".".$se;
        }
        
        # Print list of variables for debugging.
        my $message = "";
        $message .= "init():\n";
        $message .= "  \$trilinosDir = $trilinosDir\n";
        $message .= "  \$buildDir = $buildDir\n";
        $message .= "  \$makeCmd = $makeCmd\n";
        $message .= "  \$noBlame = $noBlame\n";
        $message .= "  \$outputDir = $outputDir\n";
        $message .= "  \$verbosity = $verbosity\n";
        $message .= "  \$logVerbosity = $logVerbosity\n";
        $message .= "  \$resultsDir = $resultsDir\n";
        $message .= "  \n";
        printMessage($message, $v3);
        
    } # init()
    
    ############################################################################
    # run()
    #
    # Moves into build-dir/ and runs make-cmd.
    #
    #   - args:     NONE
    #
    #   - returns:  NONE
    #

    sub run {
        
        # cd to packages directory in build directory.
        chdir $buildDir or die "cannot chdir to $buildDir, died";
        
        # Print progress message.
        my $message = "Compiling Trilinos...";
        $message = sprintf("%-25s", $message);
        printMessage($message, $v1+$v2+$v3);
        
        # Capture start time.
        my $startSeconds = time();
        
        # Run make.      
        my $makeOutput = `$makeCmd 2>&1`;
        my $makeExitStatus = $?;
        
        # Capture and format make stop time.
        (my $se, my $mn, my $hr, my $da, my $mo, my $yr) = (localtime)[0..5];
        $yr = sprintf("%02d", $yr % 100);
        $mo = sprintf("%02d", $mo+1);
        $da = sprintf("%02d", $da);
        $hr = sprintf("%02d", $hr);
        $mn = sprintf("%02d", $mn);
        $se = sprintf("%02d", $se);
        my $makeStopTime = $yr."-".$mo."-".$da." ".$hr.":".$mn.":".$se;
        my $stopSeconds = time();            
        my $runSeconds = $stopSeconds - $startSeconds;
        
        # Detect broken package.
        my $madePackages= "NONE";
        my $brokenPackage = "NONE";
        my $brokenSubdir = "NONE";
        if ($noBlame) {
            $madePackages = "see output";
            $brokenPackage = "see output";
            $brokenSubdir = "see output";
            $message = sprintf("%-30s", " broken: see output ");
            printMessage($message, $v1+$v2+$v3);
            $message = "! FAILED ";
            printMessage($message, $v1+$v2+$v3);
        } elsif ($makeCmd =~ m/tests/ || $makeCmd =~ m/examples/) {
            $madePackages = detectMadePackagesTE($makeOutput);
            if ($makeExitStatus) {
                $brokenPackage = detectBrokenPackageTE($makeOutput);
                if ($makeCmd =~ m/tests/) { $brokenSubdir = "tests"; }
                if ($makeCmd =~ m/examples/) { $brokenSubdir = "examples"; }
                $message = sprintf("%-30s", " broken: $brokenPackage:$brokenSubdir ");
                printMessage($message, $v1+$v2+$v3);
                $message = "! FAILED ";
                printMessage($message, $v1+$v2+$v3);
            } else {
                $message = sprintf("%-30s", "");
                printMessage($message, $v1+$v2+$v3);
                $message = "  passed ";
                printMessage($message, $v1+$v2+$v3);
            }
        } else {  
            $madePackages = detectMadePackages($makeOutput);
            if ($makeExitStatus) {
                ($brokenPackage, $brokenSubdir) = detectBrokenPackage($makeOutput);
                $message = sprintf("%-30s", " broken: $brokenPackage/$brokenSubdir ");
                printMessage($message, $v1+$v2+$v3);
                $message = "! FAILED ";
                printMessage($message, $v1+$v2+$v3);
            } else {
                $message = sprintf("%-30s", "");
                printMessage($message, $v1+$v2+$v3);
                $message = "  passed ";
                printMessage($message, $v1+$v2+$v3);
            }
        }
        
        # Trim $makeOutput.
        $makeOutput = substr ($makeOutput, -800000);
        
        # Finish printing pass/fail message.
        $message = ($runSeconds==0?"<1":$runSeconds);
        $message = sprintf("%6s", $message);
        printMessage($message, $v1+$v2+$v3);
        $message = " second".($runSeconds>1?"s":"")."\n";
        printMessage($message, $v1+$v2+$v3);
        
        # Grab general information values for inclusion in machine info file. 
        my $hostName = "";          my $dnsName = "";           my $ipAddress = "";
        my $operatingSystem = "";   my $kernelName = "";        my $kernelRelease = "";
        my $kernelVersion = "";     my $processor = "";         my $machineHardware = "";
        my $hardwarePlatform = "";  my $badCmd = 0;
        $badCmd = system('hostname -s > /dev/null 2>&1');
        if (!$badCmd) {
            chomp($hostName=`hostname -s`);
        } else {
            $badCmd = system('uname -n > /dev/null 2>&1');
            if (!$badCmd) { chomp($hostName=`uname -n`); }
        }
        $badCmd = system('hostname -d > /dev/null 2>&1');
        if (!$badCmd) { chomp($dnsName=`hostname -d`); }
        $badCmd = system('hostname -i > /dev/null 2>&1');
        if (!$badCmd) { chomp($ipAddress=`hostname -i`); }
        $badCmd = system('uname -o > /dev/null 2>&1');
        if (!$badCmd) { chomp($operatingSystem=`uname -o`); }
        $badCmd = system('uname -s > /dev/null 2>&1');
        if (!$badCmd) { chomp($kernelName=`uname -s`); }
        $badCmd = system('uname -r > /dev/null 2>&1');
        if (!$badCmd) { chomp($kernelRelease=`uname -r`); }
        $badCmd = system('uname -v > /dev/null 2>&1');
        if (!$badCmd) { chomp($kernelVersion=`uname -v`); }
        $badCmd = system('uname -p > /dev/null 2>&1');
        if (!$badCmd) { chomp($processor=`uname -p`); }
        $badCmd = system('uname -m > /dev/null 2>&1');
        if (!$badCmd) { chomp($machineHardware=`uname -m`); }
        $badCmd = system('uname -i > /dev/null 2>&1');
        if (!$badCmd) { chomp($hardwarePlatform=`uname -i`); }
        
        # Grab the repository branch tag
        my $branchTag = "";
        my $homeDirContents = `ls $trilinosDir`;
        if ($homeDirContents =~ m/CVS/) {
            my $cvsDirContents = `ls $trilinosDir/CVS`;
            if ($cvsDirContents =~ m/Tag/) {
                $branchTag = `cat $trilinosDir/CVS/Tag`;
                $branchTag =~ s/^T//;
            } else {
                $branchTag = "development";
            }      
        } else {
            $branchTag = "unknown";
        }
        
        # Create results file.
        my $resultsFile = "$resultsDir/$runStartTimeForFilename-make.txt";
        open (RESULTS, ">$resultsFile") 
            or die "can't open make result file $resultsFile for writing, died";
        
        my $resultsString = "";
        
        $resultsString .= "HOST_NAME            = $hostName\n";
        $resultsString .= "DNS_NAME             = $dnsName\n";
        $resultsString .= "IP_ADDRESS           = $ipAddress\n";
        $resultsString .= "OPERATING_SYSTEM     = $operatingSystem\n";
        $resultsString .= "KERNEL_NAME          = $kernelName\n";
        $resultsString .= "KERNEL_RELEASE       = $kernelRelease\n";
        $resultsString .= "KERNEL_VERSION       = $kernelVersion\n";
        $resultsString .= "PROCESSOR            = $processor\n";
        $resultsString .= "MACHINE_HARDWARE     = $machineHardware\n";
        $resultsString .= "HARDWARE_PLATFORM    = $hardwarePlatform\n";
        $resultsString .= "\n";
        $resultsString .= "TRILINOS_DIR         = $trilinosDir\n";
        $resultsString .= "BRANCH_TAG           = $branchTag\n";
        $resultsString .= "\n";
        $resultsString .= "BUILD_DIR            = $buildDir\n";
        $resultsString .= "\n";
        $resultsString .= "MAKE_COMMAND         = $makeCmd\n";
        $resultsString .= "\n";
        $resultsString .= "START_TIME           = $runStartTime\n";
        $resultsString .= "STOP_TIME            = $makeStopTime\n";
        $resultsString .= "RUN_TIME             = ".($runSeconds==0?"<1":$runSeconds)." seconds\n";
        $resultsString .= "\n";
        $resultsString .= "EXIT_STATUS          = $makeExitStatus\n";
        $resultsString .= "RESULT               = ".($makeExitStatus==0?"pass":"fail")."\n";
        $resultsString .= "\n";
        $resultsString .= "BROKEN_PACKAGE       = $brokenPackage\n";
        $resultsString .= "BROKEN_SUBDIR        = $brokenSubdir\n";
        $resultsString .= "\n";
        $resultsString .= "MADE_PACKAGES {{{\n\n$madePackages\n\n}}}\n";
        $resultsString .= "\n";
        $resultsString .= "MAKE_OUTPUT {{{\n\n$makeOutput\n\n}}}\n";
        
        print RESULTS $resultsString;
        
        close RESULTS;
        
        if ($makeExitStatus) {
            return 6;
        } else {
            return 0;
        }
        
    } # run()
    
    ############################################################################
    # detectBrokenPackage()
    #
    # Figure out which package failed to compile.
    #
    #   - args:     $makeOutput     (standard out and standard error from make)
    #
    #   - returns:  $brokenPackage  (name of the package that failed to compile)
    #

    sub detectBrokenPackage {
        my $makeOutput = $_[0];
        
        # Grab the last instance of package-level "built successfully" banner.
        my $foundPackageBanner = 0;
        my $lastSuccessfulPackage = "";
        my $packageBannerPosition = 0;
        if ($makeOutput =~ m/(.*)Trilinos package (\w*?) built successfully/ms) {
            $foundPackageBanner = 1;
            $packageBannerPosition = length($1);
            $lastSuccessfulPackage = $2;        
            $lastSuccessfulPackage = lc($lastSuccessfulPackage);
        }
        
        # Grab the last instance of subdir-level "built successfully" banner.
        my $foundSubdirBanner = 0;
        my $lastSuccessfulSubdirPackage = "";
        my $lastSuccessfulSubdir = "";
        my $subdirBannerPosition = 0;
        if ($makeOutput =~ m/(.*)Trilinos package (\w*?) subdirectory (\w*?) built successfully/ms) {
            $foundSubdirBanner = 1;
            $subdirBannerPosition = length($1);
            $lastSuccessfulSubdirPackage = $2;
            $lastSuccessfulSubdirPackage = lc($lastSuccessfulSubdirPackage);
            $lastSuccessfulSubdir = $3;
            $lastSuccessfulSubdir = lc($lastSuccessfulSubdir);
        }
        
        ########################################################################
        # Between the package banners and the subdir banners, there are four
        # possible situations:
        #        
        # 1. no subdir banner, no package banner:
        #                       first enabled package, first enabled subdir
        #
        # 2. subdir banner(s), no package banner:
        #                       subdir banner package, subsequent enabled subdir
        #
        # 3. subdir banner(s), package banner(s), package banner last:
        #                       subsequent enabled package, first enabled subdir
        #
        # 4. subdir banner(s), package banners(s), subdir banner last :
        #                       subdir banner package, subsequent enabled subdir
        #
        
        # If there was no subdir banner, or if a package banner was the last
        # banner seen, we'll need information from the packages-level Makefile,
        # so parse packages-level Makefile, extracting the list of enabled
        # packages and the package-subdir order.  We could gather this
        # information only in these two cases, but it seems harmless enough
        # to just obtain it regardless.
        
        my @enabledPackages = ();
        my @packageOrder = ();
        
        # Parse packages/Makefile, compiling a list of enabled packages.
        my $packagesMakefile = "$buildDir/packages/Makefile";
        open (PACKAGES_MAKEFILE, "<$packagesMakefile") or die "can't open $packagesMakefile";
        undef $/;                       # undefine input record separator
        my $file=<PACKAGES_MAKEFILE>;   # copy entire file
        $/ = "\n";                      # restore it to default newline
        close PACKAGES_MAKEFILE;
        
        @enabledPackages = $file =~ m/^(\w+)_SUBDIR = /gm;
        
        for (my $i=0; $i<scalar(@enabledPackages); $i++) {
            $enabledPackages[$i] = lc($enabledPackages[$i]);
        }

        my $subdirKey = "SUBDIRS";
        if ($file =~ m/^SUBDIRS = \$\(PACKAGE_NAMES\)$/m) {
            $subdirKey = "PACKAGE_NAMES";
        }
        
        # Parse packages/Makefile, extracting subdir list.
        open (PACKAGES_MAKEFILE, "<$packagesMakefile")
            or die "can't open $packagesMakefile";
        my $continue = 0;
        while (my $line = <PACKAGES_MAKEFILE>) {
            $line =~ s/^\s*//;  # trim leading spaces
            $line =~ s/\s*$//;  # trim trailing spaces
            if ($line =~ m/^$subdirKey = .*?$/) {
                $continue = 1;
            }
            if ($continue) {
                while ($line =~ m/\$\((\w*?)\)/) {
                    my $package = $1;
                    $line =~ s/\$\($package\)//;    
                    $package =~ s/_SUBDIR//;
                    $package = lc($package);
                    push (@packageOrder, $package);
                }
            }
            if ($continue && $line =~ m/^.*?\\$/) {
                $continue = 1;
            } else {
                $continue = 0;
            }
        } # while ($line)
        close PACKAGES_MAKEFILE;
        
        ########################################################################
        # At this point, we have all the information we need to determine which
        # package the failure occurred in.  From there we will then proceed
        # to determine which subdir within that package the failure occurred.
        
        my $brokenPackage = "UNKNOWN";
        
        #=======================================================================
        # Case 1:
        if (!$foundSubdirBanner && !$foundPackageBanner) {
            # Step through the packageOrder array, looking for the first
            # enabled package.
            my $foundBrokenPackage = 0;
            my $i = 0;
            while ($i < scalar(@packageOrder) && !$foundBrokenPackage) {
                if (!$foundBrokenPackage) {
                    foreach my $enabledPackage (@enabledPackages) {
                        if ($packageOrder[$i] eq $enabledPackage) {                        
                            $foundBrokenPackage = 1;
                            $brokenPackage = $enabledPackage;
                            last;
                        }
                    } # foreach
                } else {
                    last;
                }
                $i++;
            } # while ()
        }
        
        #=======================================================================
        # Case 2 or Case 4:
        if (($foundSubdirBanner && !$foundPackageBanner) ||
            ($foundSubdirBanner && $foundPackageBanner && $subdirBannerPosition > $packageBannerPosition)) {
            $brokenPackage = $lastSuccessfulSubdirPackage;
        }
        
        #=======================================================================
        # Case 3:
        if ($foundSubdirBanner && $foundPackageBanner && $packageBannerPosition > $subdirBannerPosition) {
            # Step through the packageOrder array, looking for the last
            # successfully-built package.  Once we find it, then we continue
            # stepping through the packageOrder array, now looking for the next
            # package that is enabled.    
            my $foundLastSuccessfulPackage = 0;
            my $foundBrokenPackage = 0;
            my $i = 0;
            while ($i < scalar(@packageOrder) && !$foundBrokenPackage) {
                if (!$foundLastSuccessfulPackage) {
                    if ($packageOrder[$i] eq $lastSuccessfulPackage) {
                        $foundLastSuccessfulPackage = 1;
                    }
                } else {
                    if (!$foundBrokenPackage) {
                        foreach my $enabledPackage (@enabledPackages) {
                            if ($packageOrder[$i] eq $enabledPackage) {                        
                                $foundBrokenPackage = 1;
                                $brokenPackage = $enabledPackage;
                                last;
                            }
                        } # foreach
                    } else {
                        last;
                    }
                } # else
                $i++;
            } # while ()
        }
        
        ########################################################################
        # Now we know the package in which the failure occurred, regardless
        # which of the four cases it is.  Now all we need to do is find the
        # subdirectory within that package in which the failure occurred.
        # To do so, we'll need the list of enabled subdirectories and their
        # build order.
        
        my @enabledSubdirs = ();
        my @subdirOrder = ();
         
        my $dirName = "";
        if (!getDirName($brokenPackage)) {
            printMessage("Missing entry in file test/utilities/packages!", $v1+$v2+$v3);
            $dirName = $brokenPackage;
        } else {
            $dirName = getDirName($brokenPackage);
        }
            
        # Parse packages/BROKEN_PACKAGE/Makefile, compiling a list of enabled 
        # subdirs.
        my $packageMakefile = "$buildDir/packages/".$dirName."/Makefile";
        open (PACKAGE_MAKEFILE, "<$packageMakefile") or die "can't open $packageMakefile";
        undef $/;                       # undefine input record separator
        $file=<PACKAGE_MAKEFILE>;       # copy entire file
        $/ = "\n";                      # restore it to default newline
        close PACKAGE_MAKEFILE;
        
        @enabledSubdirs = $file =~ m/^(\w+?)_SUBDIR = \w+$/gm;
        for (my $i=0; $i<scalar(@enabledSubdirs); $i++) {
            $enabledSubdirs[$i] = lc($enabledSubdirs[$i]);
        }
        
        # Parse packages/BROKEN_PACKAGE/Makefile, extracting subdir list.
        open (PACKAGE_MAKEFILE, "<$packageMakefile")
            or die "can't open $packageMakefile";
        $continue = 0;
        while (my $line = <PACKAGE_MAKEFILE>) {
            $line =~ s/^\s*//;  # trim leading spaces
            $line =~ s/\s*$//;  # trim trailing spaces
            if ($line =~ m/^SUBDIRS = .+?$/) {
                $continue = 1;
            }
            if ($continue) {
                $line =~ s/SUBDIRS = //;
                while ($line =~ m/((?:\w|\$|\(|\))+)/) {
                    my $subdir = $1;
                    if ($subdir =~ m/\$\((\w+_SUBDIR)\)/) {
                        $subdir = $1;
                        $line =~ s/\$\($subdir\)//;
                        $subdir =~ s/_SUBDIR//;
                        $subdir = lc($subdir);
                        push (@subdirOrder, $subdir);
                    } else {
                        $line =~ s/$subdir//; 
                        $subdir = lc($subdir);
                        push (@enabledSubdirs, $subdir);
                        push (@subdirOrder, $subdir);                                               
                    }
                }
            }
            if ($continue && $line =~ m/^.*?\\$/) {
                $continue = 1;
            } else {
                $continue = 0;
            }
        } # while ($line)
        close PACKAGE_MAKEFILE;
        
        ########################################################################
        # Now that we know which subdirectories were enabled and their build
        # order, we can figure out in which subdirectory the failure occurred
        # for each case.
        
        my $brokenSubdir = "UNKNOWN";
        
        #=======================================================================
        # Case 1 or Case 3:
        if ((!$foundSubdirBanner && !$foundPackageBanner) ||
            ($foundSubdirBanner && $foundPackageBanner && $packageBannerPosition > $subdirBannerPosition)) {
            # Step through the subdirOrder array, looking for the first
            # enabled subdir.
            my $foundBrokenSubdir = 0;        
            my $i = 0;
            while ($i < scalar(@subdirOrder) && !$foundBrokenSubdir) {
                if (!$foundBrokenSubdir) {              
                    foreach my $enabledSubdir (@enabledSubdirs) {
                        if ($subdirOrder[$i] eq $enabledSubdir) {                        
                            $foundBrokenSubdir = 1;
                            $brokenSubdir = $enabledSubdir;
                            last;                        
                        }
                    } # foreach
                } else {
                    last;
                }
                $i++;            
            } # while ()                    
        }
        
        #=======================================================================
        # Case 2 or Case 4:
        if (($foundSubdirBanner && !$foundPackageBanner) ||
            ($foundSubdirBanner && $foundPackageBanner && $subdirBannerPosition > $packageBannerPosition)) {
            # Step through the subdirOrder array, looking for the last
            # successfully-built subdir.  Once we find it, then we continue
            # stepping through the subdirOrder array, now looking for the next
            # subdir that is enabled.    
            my $foundLastSuccessfulSubdir = 0;
            my $foundBrokenSubdir = 0;        
            my $i = 0;
            while ($i < scalar(@subdirOrder) && !$foundBrokenSubdir) {            
                if (!$foundLastSuccessfulSubdir) {
                    if ($subdirOrder[$i] eq $lastSuccessfulSubdir) {
                        $foundLastSuccessfulSubdir = 1;
                    }
                } else {
                    if (!$foundBrokenSubdir) {              
                        foreach my $enabledSubdir (@enabledSubdirs) {                        
                            if ($subdirOrder[$i] eq $enabledSubdir) {                        
                                $foundBrokenSubdir = 1;
                                $brokenSubdir = $enabledSubdir;
                                last;                        
                            }
                        } # foreach
                    } else {
                        last;
                    }
                } # else
                $i++;            
            } # while ()
        }
        
        ########################################################################
        # Return the broken package and subdirectory
        
        return ($brokenPackage, $brokenSubdir);
           
    } # detectBrokenPackage()
    
    ############################################################################
    # detectBrokenPackageTE()
    #
    # Figure out which package failed to compile for a `make tests` or `make 
    # examples`.
    #
    #   - args:     $makeOutput     (standard out and standard error from make)
    #
    #   - returns:  $brokenPackage  (name of the package that failed to compile)
    #

    sub detectBrokenPackageTE {
        my $makeOutput = $_[0];
        
        # Grab the last banner.
        my $brokenPackage = "UNKNOWN";
        if ($makeOutput =~ m/.*^(Now|Finished) building (\w*?) (tests|examples)\.$/ms) {
            $brokenPackage = $2;        
            $brokenPackage = lc($brokenPackage);
        }
        
        return $brokenPackage;
           
    } # detectBrokenPackageTE()

    ############################################################################
    # detectMadePackagesTE()
    #
    # Compile a list of the packages that successfully compiled.
    #
    #   - args:     $makeOutput     (standard out and standard error from make)
    #
    #   - returns:  $madePackages   (list of packages that compiled successfully)
    #

    sub detectMadePackagesTE {
        my $makeOutput = $_[0];

        my @builtPackages= ();

        if ($makeCmd =~ m/tests/) {
            my @builtStrings = $makeOutput =~ m/(Finished building \w+? tests.)/g;
            foreach my $builtString (@builtStrings) {
                my $package = "";
                if ($builtString && $builtString =~ m/Finished building (\w+?) tests./) {
                    $package = lc($1);
                    push (@builtPackages, $package);
                }
            }
        } elsif ($makeCmd =~ m/examples/) {
            my @builtStrings = $makeOutput =~ m/(Finished building \w+? examples.)/g;
            foreach my $builtString (@builtStrings) {
                my $package = "";
                if ($builtString && $builtString =~ m/Finished building (\w+?) examples./) {
                    $package = lc($1);
                    push (@builtPackages, $package);
                }
            }
        }
        
        # Create list of made packages and subdirs
        my $builtPackagesString = ""; 
        foreach my $package (@builtPackages) {
            $builtPackagesString .= "$package; ";
        }  
        
        return $builtPackagesString;
           
    } # detectMadePackagesTE()
    
    ############################################################################
    # detectMadePackages()
    #
    # Compile a list of the packages that successfully compiled.
    #
    #   - args:     $makeOutput     (standard out and standard error from make)
    #
    #   - returns:  $madePackages   (list of packages that compiled successfully)
    #

    sub detectMadePackages {
        my $makeOutput = $_[0];
        
        my @madeSubdirOrder = ();
        my %madeSubdirsHoA = ();
        
        # Compile list of all "Trilinos package PACKAGE built successfully" strings.        
        my @builtStrings = $makeOutput =~ m/(Trilinos package \w+? built successfully)/g;        
        foreach my $builtString (@builtStrings) {
                            
            # Grab package name
            my $package = "";
            if ($builtString && $builtString =~ m/Trilinos package (\w+?) built successfully/) {
                $package = lc($1);
                push (@madeSubdirOrder, $package);
                if (!exists $madeSubdirsHoA{$package}) {
                    $madeSubdirsHoA{$package} = ();
                }
            }
            
        }
        
        # Compile list of all "Trilinos package PACKAGE subdirectory SUBDIR built successfully" strings. 
        my @subdirBuiltStrings = $makeOutput =~ m/(Trilinos package \w+? subdirectory \w+? built successfully)/g;
        foreach my $subdirBuiltString (@subdirBuiltStrings) {
        
            # Grab package name
            my $package = "";
            my $subdir = "";
            if ($subdirBuiltString =~ m/Trilinos package (\w+?) subdirectory (\w+?) built successfully/) {
                $package = lc($1);
                $subdir = lc($2);
                if (exists $madeSubdirsHoA{$package}) {
                    push (@{$madeSubdirsHoA{$package}}, $subdir);
                }
            }
        
        }
        
        # Create list of made packages and subdirs
        my $madePackages = ""; 
        foreach my $package (@madeSubdirOrder) {
            my @subdirs = ();
            foreach my $subdir (@{$madeSubdirsHoA{$package}}) {
                if ($subdir eq "python") { push (@subdirs, "p"); }
                if ($subdir eq "thyra") { push (@subdirs, "t"); }
            }
            my $subdirsString = join(",", @subdirs);                
            $madePackages .= "$package($subdirsString); ";
        }  
        
        return $madePackages;
           
    } # detectMadePackages()
    
    ############################################################################
    # getDirName()
    #
    # Given the standardized name of a package, return the directory name.
    #
    #   - args:     $packageName    (standardized package name)
    #
    #   - returns:                  (directory name)
    #

    sub getDirName {
        my $packageName = $_[0];       
        
        my $filename = "$trilinosDir/commonTools/test/utilities/packages";        
        open (IN_FILE, "<$filename") or die "can't open $filename";
        
        # Read definition file into one string.
        undef $/;               # undefine input record separator
        my $file=<IN_FILE>;     # copy entire file
        $/ = "\n";              # restore it to default newline
        close IN_FILE;
        
        # Clean up file.
        $file =~ s/#.*$//mg;        # remove everything after any #
        $file =~ s/\n\s*\n/\n/g;    # remove any blank lines
        $file =~ s/^\s*//mg;        # remove leading spaces
        $file =~ s/\s*$//mg;        # remove trailing spaces
        
        # break packages file into groups
        my %dirNames = $file =~ m/STD_NAME:\s*(\w+),\s*DIR_NAME:\s*(\w+)/gs;
        
        return $dirNames{$packageName}
                           
    } # getDirName()
    
    ############################################################################
    # cleanUp()
    #
    # Clean up environment variables, temp files, etc.
    #
    #   - args:     NONE
    #
    #   - returns:  NONE
    #

    sub cleanUp {
    
        # Currently, there is nothing to clean up, but I will leave this
        # subroutine here for potential future use.
    
    } # cleanUp()
    
    ############################################################################
    # printMessage()
    #
    # Prints an event if the verbosity is set.
    #
    #   - args:     $message        (message to be printed)
    #               $level          (verbosity level of message)
    #
    #   - returns:  NONE
    #

    sub printMessage {
        my $message = $_[0];
        my $level = $_[1];
        
        if ($verbosity & $level) {
            print $message;
        }
        
        if ($logVerbosity & $level) {
            my $log = $resultsDir."/log.txt";
            open (LOG, ">>$log")
                or die "can't open $log";
            print LOG $message;
            close LOG;
        }
    } # printMessage()

    ############################################################################
    # printHelp()
    #
    # Prints help output.
    #
    #   - args:     NONE
    #
    #   - returns:  NONE
    #

    sub printHelp {
        print "runmake - The Trilinos Make Utility\n";
        print "\n";
        print "Usage:  perl runmake --trilinos-dir=/home/user/Trilinos --build-dir=MPI\n";
        print "\n";
        print "Options:\n";
        print "\n";
        print "  --trilinos-dir=DIR     Specify the absolute path to the top-level Trilinos\n";
        print "                         directory that contains this program.\n";
        print "                         Example: /home/user/Trilinos\n";
        print "                         REQUIRED.\n";
        print "\n";
        print "  --build-dir=DIR        Specify the name of the build directory that contains\n";
        print "                         the configured source you would like compile.  If a\n";
        print "                         relative path is given, it is assume to be in the\n";
        print "                         given Trilinos directory.\n";
        print "                         REQUIRED.\n";
        print "\n";
        print "  --make-cmd=COMMAND     Specify the make command for this system.  This can\n";
        print "                         be used to specify make flags, \"make -j4\", for\n";
        print "                         example\n";
        print "                         Default: \"make\"\n";
        print "\n";
        print "  --output-dir=DIR       Specify the directory in which to create the directory\n";
        print "                         containing the results.\n";
        print "                         Default: \".\"\n";
        print "\n";
        print "  --verbosity=LEVEL      0 = no non-fatal ouput (same as --quiet)\n";
        print "                         1 = normal output (default)\n";
        print "                         2 = level 2 verbosity\n";
        print "                         3 = level 3 verbosity\n";
        print "\n";
        print "  --log-verbosity=LEVEL  0 = no log\n";
        print "                         1 = normal output (default)\n";
        print "                         2 = level 2 verbosity\n";
        print "                         3 = level 3 verbosity\n";
        print "\n";
        print "  --quiet                Produce no non-fatal output.\n";
        print "\n";
        print "  --help                 Print this help output and exit.\n";
        print "\n";
        print "Notes:\n";
        print "  - For more information, see README-runmake in\n";
        print "    Trilinos/commonTools/test/utilities/\n";
        print "    or visit http://software.sandia.gov/trilinos/developer/\n";
        print "\n";
    } # printHelp()
