#!/usr/bin/env perl
##**************************************************************
##
## Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
## University of Wisconsin-Madison, WI.
## 
## Licensed under the Apache License, Version 2.0 (the "License"); you
## may not use this file except in compliance with the License.  You may
## obtain a copy of the License at
## 
##    http://www.apache.org/licenses/LICENSE-2.0
## 
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##
##**************************************************************


use POSIX ":sys_wait_h";

	##these are the files that need to be staged on the remote machine
$config_name = "glidein_condor_config";
$startup_name = "glidein_startup";
@DAEMONS = ( "condor_master", "condor_startd", "condor_starter",
			 "condor_starter.std" );

if ( ( $error = parse_command_line() )
		|| ( $error = gather_info() ) )
{
	print STDERR "$error\n";
	exit( 1 );
}

if ( $gen_config ) {
	my $glidein_condor_config = "glidein_condor_config$suffix";
	gen_main_config($glidein_condor_config);
	if ( ! -f $glidein_condor_config ) {
	    print STDERR "Failed to create $glidein_condor_config.\n";
	    exit( 1 );
	}
	print "Generated $glidein_condor_config\n";
}
if ( $gen_glidein_startup ) {
	my $glidein_startup = "glidein_startup$suffix";
	gen_glidein_startup($glidein_startup);
	if ( ! -f $glidein_startup ) {
	    print STDERR "Failed to create $glidein_startup.\n";
	    exit( 1 );
	}
	print "Generated $glidein_startup\n";
}

if ( $gen_config || $gen_glidein_startup ) {
    exit( 0 );
}

if ( $#ARGV == -1 ) {
    if ( ($run_here && $setup_here) ||
	 ($run_here && $runonly) ||
	 ($setup_here && $setuponly)) {
	@CONTACTS = (`hostname`)
    }
    else {
	print STDERR "No target resource specified.  Use -help for usage information.\n";
	exit( -1 );
    }
}

foreach $CONTACT ( @CONTACTS ) {
	if ( host_info() ) {
		next;
	}

	if ( !$runonly ) {
		if ( $error = do_remote_setup() ) {
			print STDERR "Error: $error\n";
			exit( 2 );
		}
	}
	if ( !$setuponly ) {
		if ( $error = do_submit() ) {
			print STDERR "Error: $error\n";
			exit( 3 );
		}
	}
}

exit( 0 );

## END OF PROGRAM

sub do_submit() {

	print "\nLaunching Glidein...\n";
	if ( $arch =~ /linux/ ) {
		$mail_arch = "/usr/bin/mail";
	}
	elsif ( $arch =~ /solaris/ ) {
		$mail_arch = "/usr/ucb/mail";
	}
	elsif ( $arch =~ /irix/ ) {
		$mail_arch = "/usr/sbin/mailx";
	}
	else {
		$mail_arch = "/bin/mail";
	}
	$GLOBUSRSL = "";
	if ( $queue ) {
		$GLOBUSRSL .= "(queue=$queue)";
	}
	if ( $project ) {
		$GLOBUSRSL .= "(project=$project)";
	}
	if ( $globus_run_time ) {
		$GLOBUSRSL .= "(maxTime=$globus_run_time)";
	}
	if ( $memory ) {
		$GLOBUSRSL .= "(maxMemory=$memory)";
	}
	if ( $cpucount > 1 ) {
			##claim N cpus, but only run one copy of the program
		$GLOBUSRSL .= "(count=$cpucount)(jobtype=single)";
	}

	$GLOBUSENV .= "CONDOR_CONFIG=$basedir/$config_name;";
	$GLOBUSENV .= "_condor_CONDOR_HOST=$CONDOR_HOST;";

	chomp( $my_hostname = `$G_HOSTNAME` );
	$GLOBUSENV .= "_condor_GLIDEIN_HOST=$my_hostname;";

	$GLOBUSENV .= "_condor_LOCAL_DIR=$localdir;";
	$GLOBUSENV .= "_condor_SBIN=$archdir;";

	$GLOBUSENV .= "_condor_CONDOR_ADMIN=$condor_admin;";

		##if submitting to Condor scheduler under Globus, don't want "queue <n>"
		##starting <n> copies, EACH asking for <n> CPUs, that would give us
		## <n>**2 CPUs....
	if ( $CONTACT =~ /condor/ ) {
		$GLOBUSENV .= "_condor_NUM_CPUS=1;";
	}
	else {
		$GLOBUSENV .= "_condor_NUM_CPUS=$cpucount;";
	}
	$GLOBUSENV .= "_condor_UID_DOMAIN=$UID_DOMAIN;";
	$GLOBUSENV .= "_condor_FILESYSTEM_DOMAIN=$UID_DOMAIN;";
	$GLOBUSENV .= "_condor_MAIL=$mail_arch;";
	if ( $idle_time != 0 ) {
		$GLOBUSENV .= "_condor_STARTD_NOCLAIM_SHUTDOWN=" . ($idle_time * 60) . ";";
	}
	if ( $memory ) {
		$kmem = $memory * 1024;
		$GLOBUSENV .= "_condor_MEMORY=$kmem;";
	}
	if ( $anybody ) {
		$GLOBUSENV .= "_condor_START=True;";
	}
	else {
		$GLOBUSENV .= "_condor_START_owner=$ENV{'LOGNAME'};";
	}
	if ( $slotcount != $cpucount ) {
		$GLOBUSENV .= "_condor_SLOT_TYPE_1=1/$slotcount;";
		$GLOBUSENV .= "_condor_NUM_SLOTS_TYPE_1=$slotcount;";
	}

		##Condor has trouble determining ARCH on some
		##high-performance HP-UX machines because they have a
		##minimal OS install.  So, set the ARCH here if we
		##know it.
	if ( $arch =~ /hppa2/ ) {
		$GLOBUSENV .= "_condor_ARCH=HPPA2;";
	}
	elsif ( $arch =~ /hppa1/ ) {
		$GLOBUSENV .= "_condor_ARCH=HPPA1;";
	}

	chomp( $skipauth = `$CONFIGVAL SKIP_AUTHENTICATION 2>/dev/null` );
	if ( $skipauth ne "" ) {
		$GLOBUSENV .= "_condor_SKIP_AUTHENTICATION=$skipauth;";
	}

	chomp( $usetcp = `$CONFIGVAL UPDATE_COLLECTOR_WITH_TCP 2>/dev/null` );
	if ( $usetcp ne "" ) {
		$GLOBUSENV .= "_condor_UPDATE_COLLECTOR_WITH_TCP=$usetcp;";
	}

	if ( $master_run_time ) {
		$GLOBUSENV .= "_condor_DaemonStopTime=DaemonStartTime+" . $master_run_time*60 . ";";
	}

	chop $GLOBUSENV;

	$arguments = "-dyn -f";
	if ( $master_run_time ) {
		$arguments .= " -r $master_run_time";
	}

	if ( $run_here ) {
	    my $glidein_run_sh = "glidein_run.sh$suffix";
	    my $commands = "#!/bin/sh\n";
	    my $shenv = $GLOBUSENV;
	    my $sh_archdir = $archdir;

	    #convert env references to sh syntax
	    $shenv =~ s/\$\(HOME\)/\${HOME}/g;
	    $sh_archdir =~ s/\$\(HOME\)/\${HOME}/g;

	    for my $env_setting (split(/;/,$shenv)) {
		my $var, $val;
		($var, $val) = split(/=/,$env_setting,2);
		$commands .= "$var=\"$val\"\n";
		$commands .= "export $var\n";
	    }
	    $commands .= "\nexec $sh_archdir/$startup_name $arguments\n";
	    if ( $gen_submit ) {
		print "Saving glidein submit file: $glidein_run_sh\n";
		open( SUBMIT, ">$glidein_run_sh" );
		print SUBMIT $commands;
		close( SUBMIT );
		chmod 0755, $glidein_run_sh;
	    } else {
		print "Running Glidein...\n";
		if( system($commands) != 0 ) {
		    return ("glidein failed to run.");
		}
	    }
	    return;
	}

		##Escape '$' in submit file entries
	$archdir =~ s/\$/\$(DOLLAR)/g;
	$GLOBUSRSL =~ s/\$/\$(DOLLAR)/g;
	$GLOBUSENV =~ s/\$/\$(DOLLAR)/g;

		##generate condor_submit description file
	my $glidein_run_submit = "glidein_run.submit$suffix";
	if ( $gen_submit ) {
		print "Saving glidein submit file: $glidein_run_submit\n";
		open( SUBMIT, ">$glidein_run_submit" );
	} else {
		my $my_grid_proxy_err = check_grid_proxy();
		if( $my_grid_proxy_err ) {
			return $my_grid_proxy_err;
		}

		print "Submitting Glidein job...\n";
		open( SUBMIT, "|$submit" );
	}
	print SUBMIT <<"EOF";
Universe = Grid

Executable = $archdir/$startup_name

Arguments = $arguments

Environment = ${GLOBUSENV}

Transfer_Executable = False

GlobusRSL = ${GLOBUSRSL}

Grid_Resource = gt2 $CONTACT

Notification = Never
Queue

EOF
	close( SUBMIT );
	if ( $gen_submit ) {
		return 0;
	} elsif( $? ) {
		return ("condor_submit failed.");
	}
}

sub check_grid_proxy {
    my $g_info;
    chomp( $g_info = `which grid-proxy-info 2>/dev/null` );
    if ( ! -x $g_info ) {
	return( "cannot find grid-proxy-info" );
    }
    if ( silently( "$g_info -exists" ) ) {
	return( "$NAME needs a valid grid proxy. (Re)run grid-proxy-init." );
    }
    if ( silently( "$g_info -exists -hours 4" ) ) {
	return( "Your grid proxy expires in less than four hours.\n"
		. "Rerun grid-proxy-init." );
    }
}

sub gather_info() {
		##ensure the needed condor programs are avail...
	chomp( $submit = `which condor_submit` );
	chomp( $condor_wait = `which condor_wait` );
	chomp( $CONFIGVAL = `which condor_config_val` );
	chomp( $STATUS = `which condor_status` );
	if ( ! ( -x $CONFIGVAL && -x $STATUS && -x $submit && -x $condor_wait) ) {
		return( "cannot find programs \"condor_config_val\" or "
			. "\"condor_status\" or \"condor_submit\" or "
			. "\"condor_wait\" needed for setup" );
	}

	if ( !defined( $condor_admin ) ) {
		chomp( $condor_admin = getlogin() . '@' . `$CONFIGVAL UID_DOMAIN`);
	}

	chomp( $CONDOR_HOST = `$CONFIGVAL CONDOR_HOST` );
	chomp( $COLLECTOR_HOST = `$CONFIGVAL COLLECTOR_HOST` );

	#NEGOTIATOR_HOST is not defined in modern config files (circa 6.7.10)
	#so if it is not defined, skip it.
	chomp( $NEGOTIATOR_HOST = `$CONFIGVAL NEGOTIATOR_HOST 2>/dev/null` );
	if ( $NEGOTIATOR_HOST eq "" ) {
	    $NEGOTIATOR_SETTING = ""
	}
	else {
	    $NEGOTIATOR_SETTING = "NEGOTIATOR_HOST = $NEGOTIATOR_HOST"
	}

	#set CONDOR_VERSION
	$TMP = `$condor_wait -version | grep CondorVersion`;
	( $_, $CONDOR_VERSION, $_) = split( / *[$: ]+/, $TMP );
	if ( $CONDOR_VERSION eq "" ) {
		return( "Can't determine Condor version.\n" );
	}

	#Find out where the glidein tarball server is
	chomp( $SERVER_URL = `$CONFIGVAL GLIDEIN_SERVER_URLS 2>/dev/null` );

	if( $SERVER_URL eq "" ) {
	    #Try reading the old deprecated config info
	    chomp( $SERVER_NAME = `$CONFIGVAL GLIDEIN_SERVER_NAME 2>/dev/null` );
	    chomp( $SERVER_DIR = `$CONFIGVAL GLIDEIN_SERVER_DIR 2>/dev/null` );
	    if ( $SERVER_NAME eq "" && $SERVER_DIR eq "" ) {
		#No info available, so tell them about the new way to config
		return( "Error, GLIDEIN_SERVER_URLS must be defined in the condor_config config file" );
	    }
	    elsif ( $SERVER_NAME eq "" ) {
		return( "Error, GLIDEIN_SERVER_NAME needs to be defined in the Condor config file" );
	    }
	    elsif ( $SERVER_DIR eq "" ) {
		return( "Error, GLIDEIN_SERVER_DIR needs to be defined in the Condor config file" );
	    }
	    $SERVER_URL = "gsiftp://$SERVER_NAME$SERVER_DIR";
	}

	#Put into canonical comma separated form if there are multiple URLs.
	$SERVER_URL = join(',',split(/[, ]+/,$SERVER_URL));

		##if archdir was specified on command line
	if ( defined( $ARCHDIR ) ) {
		$archdir = $ARCHDIR;
	}

		## globus programs
	chomp( $G_HOSTNAME = `which globus-hostname 2>/dev/null` );
	if ( ! -x $G_HOSTNAME ) {
	    chomp( $G_HOSTNAME = `which hostname 2>/dev/null` );
	    if ( ! -x $G_HOSTNAME ) {
		return( "Error, cannot find globus-hostname or hostname" );
	    }
	}

		##also, ensure GLOBUS_LOCATION and HOME are set in ENV
	if ( !defined( $ENV{'HOME'} ) ) {
		return( "$NAME: $HOME not in your ENV" );
	}

	if( $config_gsi ) {
	    if( $install_gsi_trusted_ca_dir eq "" ) {
		return( "ERROR: GSI setup requires -install_gsi_trusted_ca_dir to be specified.\nExample: -install_gsi_trusted_ca_dir /etc/grid-security/certificates" );
	    }
	    if( $install_gsi_gridmap eq "" ) {
		return( "ERROR: GSI setup requires -install_gsi_gridmap to be specified.\nThis should be a grid-mapfile containing entries for authorized users." );
	    }
	    if( $gsi_daemon_name eq "" ) {
		return( "ERROR: GSI setup requires -gsi_daemon_name to be specified.  This should be set to your grid certificate name." );
	    }
	}

	return( 0 );
}

sub usage() {
		#print out the command line options
	print STDERR "Usage: $NAME [options] [<globus contact>]\n";
	print STDERR "\tWhere <globus contact> is of form:\n";
	print STDERR "\t<hostname> | <hostname>/jobmanager-<scheduler> | <full contact string>\n";

print STDERR "	Valid options:\n"
. "   -basedir <name>         Use <name> as remote base directory\n"
. "                            [default: \$(HOME)/Condor_glidein]\n"
. "   -archdir <name>         Specify remote dir for daemon executables\n"
. "                            [default is built from system config info]\n"
. "   -localdir <name>        Use <name> as remote base for log/execute dirs\n"
. "                            [default on remote host: <basedir>/local]\n"
. "   -contactfile <file>     Specify file from which to read Globus contacts\n"
. "   -setuponly|-runonly     Setup and exit  |  No setup, just run\n"
. "   -forcesetup             During setup, force copying (i.e. updating) of files\n"
. "   -setup_jobmanager <X>   Jobmanager to use for setup.\n"
. "   -setup_here             Run setup procedure directly, rather than remotely.\n"
. "   -run_here               Run glidein directly, rather than remotely.\n"
. "   -arch <architecture>    Condor architecture to install on remote fesource.\n"
. "   -queue <queue>          Queue to use on Globus scheduler\n"
. "   -project <project>      Project to use on Globus resource\n"
. "   -memory <Mbytes>        Amount of memory to request\n"
. "   -count <CPU count>      Number of CPUs to request [default=1]\n"
. "   -slots <Slot count>     Number of slots to split up allocated resources\n"
. "                             into [default=<CPU count>]\n"
. "   -vms <Slot count>       Deprecated way to specify -slots\n"
. "   -runtime <mins>         Maximum run time on Globus resource\n"
. "   -idletime <mins>        Remote daemons exit gracefully after <mins>\n"
. "                            without running a Condor job [default=20]\n"
. "   -anybody                Allow any user to run jobs on your glideins\n"
. "   -admin <email address>  Who to email with problems [default=<you>]\n"
. "   -genconfig              Generate glidein_condor_config; do not submit\n"
. "   -useconfig <file>       Use the specifed config file\n"
. "   -genstartup             Generate glidein_startup; do not submit\n"
. "   -usestartup <file>      Use the specifed glidein startup file\n"
. "   -gensubmit              Generate submit files; do not submit\n"
. "   -suffix X               Suffix to append to submit file names.\n"
. "   -edg                    Use EDG MDS schema to query GRIS\n"
. "   -install_gsi_trusted_ca_dir <path>   Copy trusted CA certs.\n"
. "   -install_gsi_gridmap <file>          Copy grid-mapfile.\n"
. "   -gsi_daemon_name <cert name>         Your grid certificate name.\n"
. "   -help                   Print this usage, see documentation for info\n";
}

sub parse_command_line() {
		#parse command line options
	($NAME = $0) =~ s/.*\///; ## strip off directory path...
	$idle_time = 20;
	$cpucount = 1;
	$slotcount = 0;
	$vmcount = 0;    ## CRUFT
	$basedir = '$(HOME)/Condor_glidein';
	##don't set default archdir because it is built below from pool version, arch
	$suffix = ".$$";
	$localdir = "";
	$use_config = "";
	$use_glidein_startup = "";
	chomp( $username = `whoami` );

	use Getopt::Long;
	$RESULT = GetOptions(
		"basedir=s" => \$basedir,
		"archdir=s" => \$ARCHDIR,
		"localdir=s" => \$localdir,
		"contactfile=s" => \$contactfile,
		"setuponly" => \$setuponly,
		"runonly" => \$runonly,
		"forcesetup" => \$force_setup,
		"queue=s" => \$queue,
		"project=s" => \$project,
		"memory=i" => \$memory,
		"count=i" => \$cpucount,
		"slots=i" => \$slotcount,
		"vms=i" => \$vmcount,
		"runtime=i" => \$globus_run_time,
		"idletime=i" => \$idle_time,
		"anybody" => \$anybody,
		"admin=s" => \$condor_admin,
		"genconfig" => \$gen_config,
		"useconfig=s" => \$use_config,
		"genstartup" => \$gen_glidein_startup,
		"usestartup=s" => \$use_glidein_startup,
		"gensubmit" => \$gen_submit,
		"edg" => \$edg_mds_schema,
		"setup_here" => \$setup_here,
		"run_here" => \$run_here,
		"setup_jobmanager=s" => \$setup_jobmanager,
		"setup_queue=s" => \$setup_queue,
		"arch=s" => \$setup_arch,
		"suffix=s" => \$suffix,
		"install_gsi_trusted_ca_dir=s" => \$install_gsi_trusted_ca_dir,
		"install_gsi_gridmap=s" => \$install_gsi_gridmap,
		"gsi_daemon_name=s" => \$gsi_daemon_name,
		"help" => \$help );
	if ( !$RESULT ) {
		usage();
		exit( 1 );
	}

	$config_gsi = 0;
	if( $install_gsi_trusted_ca_dir ne ""
	    || $install_gsi_gridmap ne ""
	    || $gsi_daemon_name ne "" ) {
	    $config_gsi = 1;
	}
	if ( $suffix ne "" && !($suffix =~ /\..*/) ) {
	    $suffix = "." . $suffix;
	}
	if ( $vmcount && $slotcount) {
	    return( "ERROR: $NAME --vms and --slots conflict, just use --slots");
	}
	if ( $vmcount ) {
	    print "WARNING: -vms is deprecated, use -slots instead\n";
	    $slotcount = $vmcount;
	}
	if ( $slotcount == 0 ) {
	    $slotcount = $cpucount;
	}
	if ( $cpucount % $slotcount ) {
	    return( "ERROR: $NAME --slots must be a factor of --count");
	}


	if ( $localdir eq "" ) {
		$localdir = "$basedir/local";
	}

	if ( $setuponly && $runonly ) {
		return( "ERROR: $NAME --setuponly and --runonly are mutually exclusive");
	}

	if ( ! $force_setup ) {
		$force_setup = 0;
	}

	if ( $globus_run_time ) {
		if ( $globus_run_time > 0 ) {
			# Tell the master to shutdown a minute before the Globus limit
			$master_run_time = $globus_run_time - 1;
		} else {
			return( "ERROR: $NAME --runtime must be positive" );
		}
	}

	if ( $help ) {
		print STDERR "Add one or more remote nodes to your local "
		             . "Condor pool\n";
		print STDERR "For detailed help, see the condor documentation "
		             . "concerning condor_glidein\n\n";
		usage();
		exit( 0 );
	}

	if ( $contactfile ) {
		open( CONTACTS, "< $contactfile" );
		@CONTACTS = <CONTACTS>;
	}
	else {
		push( @CONTACTS, join( ' ', @ARGV ) );
	}
	chomp( @CONTACTS );

	return( 0 );
}

sub silently {
		#get rid of unsightly output from system commands
		#expects a list compatible with system()
	if ( $pid = fork() ) {
		#parent
		waitpid $pid, 0;
	}
	else {
		#child
		close STDOUT;
		close STDERR;
		open( STDOUT, ">/dev/null" );
		open( STDERR, ">/dev/null" );
		exec( @_ );
	}
	return $?;
}

sub gen_glidein_startup {
	my $filename = shift(@_);
	open( GLIDEIN_STARTUP, ">$filename")
	    or die "Can't write to $filename: $!";

	print GLIDEIN_STARTUP <<"EOF";
#!/bin/sh

if [ "\$_condor_LOCAL_DIR" = "." ]; then
  export _condor_LOCAL_DIR=\$(pwd)
fi

#If we have an X509 user proxy, use it as the Condor daemon proxy.
if [ "\$_condor_GSI_DAEMON_PROXY" = "" ] && [ -a "\$X509_USER_PROXY" ]; then
  export _condor_GSI_DAEMON_PROXY="\$X509_USER_PROXY"
fi

exec \${_condor_SBIN}/condor_master "\$@"
EOF
	close GLIDEIN_STARTUP
	    or die "Error writing to $filename: $!";

	chmod 0755, $filename;
}

sub gen_main_config {
	my $filename = shift(@_);
	##generate the config files
	open( CONDOR_CONFIG, ">$filename")
	    or die "Can't write to $filename: $!";

	print CONDOR_CONFIG <<"EOF";

LOG = \$(LOCAL_DIR)/log
EXECUTE = \$(LOCAL_DIR)/execute
HOSTALLOW_ADMINISTRATOR = \$(HOSTNAME), \$(GLIDEIN_HOST)

MAIL = /bin/mail
DAEMON_LIST = MASTER, STARTD
MASTER_ADDRESS_FILE = \$(LOG)/.master_address
MASTER = \$(SBIN)/condor_master
MASTER_LOG = \$(LOG)/MasterLog
STARTD = \$(SBIN)/condor_startd
STARTER = \$(SBIN)/condor_starter
PROCD = \$(SBIN)/condor_procd
PROCD_LOG = \$(LOG)/ProcLog
PROCD_ADDRESS = \$(LOCK)/procd_address

STARTER_STD = \$(SBIN)/condor_starter.std
STARTER_LIST = STARTER, STARTER_STD

USE_NFS = False
USE_CKPT_SERVER = False
$NEGOTIATOR_SETTING
COLLECTOR_HOST = $COLLECTOR_HOST

LOCK = \$(LOG)
STARTD_LOG = \$(LOG)/StartLog
STARTER_LOG = \$(LOG)/StarterLog

STARTD_DEBUG = D_COMMAND

SUSPEND = False
CONTINUE = True
PREEMPT = False
WANT_VACATE = False
WANT_SUSPEND = True
SUSPEND_VANILLA = False
WANT_SUSPEND_VANILLA = True
KILL = False
IS_GLIDEIN = True
STARTD_EXPRS = IS_GLIDEIN, START, DaemonStopTime

MASTER_WAITS_FOR_GCB_BROKER = False
EOF
	if( $config_gsi ) {
	    print CONDOR_CONFIG <<"EOF";
SEC_DEFAULT_AUTHENTICATION = REQUIRED
SEC_DEFAULT_AUTHENTICATION_METHODS = GSI
GSI_DAEMON_DIRECTORY = \$(SBIN)/../grid-security
GRIDMAP = \$(GSI_DAEMON_DIRECTORY)/grid-mapfile
GSI_DAEMON_NAME = $gsi_daemon_name
EOF
	}

	close CONDOR_CONFIG
	    or die "Error writing to $filename: $!";
}

sub replace_jobmanager
{
    my $contact = shift();
    my $jobmanager = shift();

    if ( $jobmanager ne "" ) {
	$jobmanager = "/" . $jobmanager
    }

    if ( grep( /jobmanager/, $contact ) ) {
	$contact =~ s|/[^:]*|$jobmanager|;
    } elsif ( grep( /:/, $contact ) < 2 ) {
	$contact .= "$jobmanager";
    } else {
	$contact =~ s|(:[^:]*)|\1$jobmanager|;
    }
    return $contact;
}

sub grid_info_search {
	my $host = shift();
	my $port = shift();
	my $query_string = shift();
	my $grid_info_search;
	chomp( $grid_info_search = `which grid-info-search 2>/dev/null` );
	if ( ! -x $grid_info_search ) {
	    print STDERR "Warning: cannot find grid-info-search\n";
	    return;
	}
	open( QUERY, "$grid_info_search -x -h $host -p $port -b $query_string|");
	my @result = <QUERY>;
	if ( !close( QUERY ) ) {
		print STDERR "Warning: failed to query GRIS on $host:$port\n";
		return;
	}
	return @result;
}

sub query_edg_mds_schema()
{
    #The remote machine is an EDG Compute Element. Use the EDG MDS
    #schema to query the GRIS server there. We're looking for the
    #architecture of the Compute Element.
    my @QUERY = grid_info_search($GLOBUSHOST,2135,"'mds-vo-name=local,o=grid' '(objectclass=MdsComputer)' Mds-Cpu-vendor Mds-Computer-platform Mds-Os-name Mds-Os-release" );
    if ( !@QUERY ) {
	return;
    }

    #Currently, the EDG MDS schema doesn't advertise anything that lets
    #us find a jobmanager service & queue that'll run immediately for
    #the setup phase, so we'll just try "jobmanager-fork".
    if( $CONTACT_SETUP_QUEUE eq "" || $CONTACT_SETUP eq "") {
	$CONTACT_SETUP_QUEUE = "unknown";
	$CONTACT_SETUP = replace_jobmanager($CONTACT,"jobmanager-fork");
    }

    #Search the query results for the architecture of the resource the
    #user wants to glidein to. In EDG land, there's only one
    #architecture for a given Compute Element (a front-end machine
    #running Globus and all it's associated back-end nodes).
    @SEARCH1 = grep( /^Mds-Computer-platform:/, @QUERY );
    @SEARCH2 = grep( /^Mds-Cpu-vendor:/, @QUERY );
    @SEARCH3 = grep( /^Mds-Os-name:/, @QUERY );
    @SEARCH4 = grep( /^Mds-Os-release:/, @QUERY );
    my $cputype = "";
    my $manufacturer = "";
    my $osname = "";
    my $osversion = "";
    if ( $#SEARCH1 >= 0 ) {
	chomp( $cputype = $SEARCH1[0] );
	$cputype =~ s/Mds-Computer-platform: //;
	chomp( $manufacturer = $SEARCH2[0] );
	$manufacturer =~ s/Mds-Cpu-vendor: //;
	chomp( $osname = $SEARCH3[0] );
	$osname =~ s/Mds-Os-name: //;
	chomp( $osversion = $SEARCH4[0] );
	$osversion =~ s/Mds-Os-release: //;
	if ( $osname =~ /[Ll]inux/ ) {
	    $osversion =~ s/(\d+\.\d+)\..*/\1/;
	}
	if ( $manufacturer =~ /GenuineIntel/ ) {
	    $manufacturer = "pc";
	}
	if ( $manufacturer =~ /AuthenticAMD/ ) {
	    $manufacturer = "pc";
	}
    }

    if($arch eq "") {
	if ( $cputype ne "" && $manufacturer ne "" && $osname ne "" &&
	 $osversion ne "" ) {
	    $arch = "${CONDOR_VERSION}-${cputype}-${manufacturer}-${osname}-${osversion}";
	}
    }
}

sub query_globus_mds_schema()
{

    #Query the GRIS server on the globus machine using the Globus MDS
    #schema. We're looking for two
    #things: which jobmanager/queue will run jobs immediately (for
    #setup), and the architecture of the jobmanager the user wants to
    #run on (not necessarily the same as the machine we're talking to).
    my @QUERY = grid_info_search($GLOBUSHOST,2135,"'mds-vo-name=local,o=grid' '(|(objectclass=MdsServiceGram)(objectclass=MdsGramJobQueue))' Mds-Software-deployment Mds-Gram-Job-Queue-dispatchtype Mds-Computer-isa Mds-Computer-manufacturer Mds-Os-name Mds-Os-release");
    if ( !@QUERY ) {
	return;
    }

    #Search the query results for an "Immediate" jobmanager/queue
    @SEARCH1 = grep( /^dn: Mds-Job-Queue-name=/, @QUERY );
    @SEARCH2 = grep( /^Mds-Gram-Job-Queue-dispatchtype:/, @QUERY );
    my $queue = "";
    my $service = "";
    for ( $i = 0; $i <= $#SEARCH1; $i++ ) {
	if ( grep( /Immediate/, $SEARCH2[$i] ) > 0 ) {
	    chomp( $queue = $SEARCH1[$i] );
	    $queue =~ s/^dn: Mds-Job-Queue-name=([^,]*),.*$/\1/;
	    chomp( $service = $SEARCH1[$i] );
	    $service =~ s/^.*Mds-Software-deployment=([^,]*),.*$/\1/;
	    last;
	}
    }

    if( $CONTACT_SETUP_QUEUE eq "" || $CONTACT_SETUP eq "") {
	$CONTACT_SETUP_QUEUE = $queue;
	$CONTACT_SETUP = replace_jobmanager($CONTACT,$CONTACT_SETUP);
    }

    #Search the query results for the architecture of the jobmanager
    #the user wants to glidein to.
    my $jobmanager;
    if ( grep( /jobmanager/, $CONTACT ) ) {
	$jobmanager = $CONTACT;
	$jobmanager =~ s|^[^/]*/([^:]*).*|\1|;
    } else {
	$jobmanager = "jobmanager";
    }
    @SEARCH1 = grep( /^Mds-Software-deployment:/, @QUERY );
    @SEARCH2 = grep( /^Mds-Computer-isa:/, @QUERY );
    @SEARCH3 = grep( /^Mds-Computer-manufacturer:/, @QUERY );
    @SEARCH4 = grep( /^Mds-Os-name:/, @QUERY );
    @SEARCH5 = grep( /^Mds-Os-release:/, @QUERY );
    my $cputype = "";
    my $manufacturer = "";
    my $osname = "";
    my $osversion = "";
    for ( $i = 0; $i <= $#SEARCH1; $i++ ) {
	if ( grep( /$jobmanager/, $SEARCH1[$i] ) > 0 ) {
	    chomp( $cputype = $SEARCH2[$i] );
	    $cputype =~ s/Mds-Computer-isa: //;
	    chomp( $manufacturer = $SEARCH3[$i] );
	    $manufacturer =~ s/Mds-Computer-manufacturer: //;
	    chomp( $osname = $SEARCH4[$i] );
	    $osname =~ s/Mds-Os-name: //;
	    chomp( $osversion = $SEARCH5[$i] );
	    $osversion =~ s/Mds-Os-release: //;
	    if ( $osname =~ /[Ll]inux/ ) {
		$osversion =~ s/(\d+\.\d+)\..*/\1/;
	    }
	    last;
	}
    }

    if($arch eq "") {
	if ( $cputype ne "" && $manufacturer ne "" && $osname ne "" &&
	       $osversion ne "" ) {
	    $arch = "${CONDOR_VERSION}-${cputype}-${manufacturer}-${osname}-${osversion}";
	}
    }
}

sub host_info() 
{
		##get rid of everything following '#' in contact entries
	$CONTACT =~ s/#.*//;

	if ( ( $CONTACT eq "" ) || !defined( $CONTACT ) ) {
		return( 1 );
	}

	$GLOBUSHOST = $CONTACT;
	$GLOBUSHOST =~ s/[:\/].*//;

	$UID_DOMAIN = $GLOBUSHOST;
	$UID_DOMAIN =~ s/[^\.]*\.//;

	$CONTACT_SETUP_QUEUE = "";
	$CONTACT_SETUP = "";
	$arch = "";

	#Explicit settings from command-line override GRIS.
	if( $setup_queue ne "" ) {
	    $CONTACT_SETUP_QUEUE = $setup_queue;
	}
	if( $setup_jobmanager ne "" ) {
	    $CONTACT_SETUP = replace_jobmanager($CONTACT,$setup_jobmanager);
	    if($CONTACT_SETUP_QUEUE eq "") {
		$CONTACT_SETUP_QUEUE = "unknown";
	    }
	}
	if( $setup_here ) {
	    if( $CONTACT_SETUP eq "" ) {
		$CONTACT_SETUP = "none";
	    }
	    if( $CONTACT_SETUP_QUEUE eq "" ) {
		$CONTACT_SETUP_QUEUE = "unknown";
	    }
	}
	if( $setup_arch ne "" ) {
	    $arch = $setup_arch;
	}

	if( $arch eq "" || \
	    $CONTACT_SETUP eq "" || \
	    $CONTACT_SETUP_QUEUE eq "")
	{
	    if ( $edg_mds_schema ) {
		query_edg_mds_schema();
	    } else {
		query_globus_mds_schema();
	    }
	}

	if( !$runonly ) {
	    if( $CONTACT_SETUP eq "" || $CONTACT_SETUP_QUEUE eq "" ) {
		print STDERR "Cannot automatically determine immediate jobmanager for setup on\n$GLOBUSHOST.\n";
		print STDERR "Try inserting something like -setup_jobmanager=jobmanager-fork\non your condor_glidein command-line.\n";
		return( 1 );
	    }
	}

	if( $arch eq "" ) {
	    print STDERR "\nPROBLEM: Can't determine architecture of $GLOBUSHOST\n";
	    print STDERR "You can bypass this by specifying the architecture on the command line.\nExample: -arch=${CONDOR_VERSION}-i686-pc-Linux-2.4\n";
	    print STDERR "For a full list of supported architectures, visit\nhttp://www.cs.wisc.edu/condor/glidein/binaries\n";
	    return( 1 );
	}

	if ( !defined( $ARCHDIR ) || $ARCHDIR eq "" ) {
		$archdir = "$basedir/$arch";
	}

	return( 0 );
}

sub get_more_output {
    my $filename = shift();
    my $position = shift();
    my $output = "";

    if(!open( GET_MORE_OUTPUT_FD, "<", $filename )) {
	return $output;
    }
    seek GET_MORE_OUTPUT_FD,${$position},0;
    my $n = read(GET_MORE_OUTPUT_FD,$output,1000000);
    ${$position} += $n;
    close(GET_MORE_OUTPUT_FD);
    return $output;
}

sub condor_submit {
    my $submit_file = shift();
    my $job_id;

    $condor_submit_pid = open(CONDOR_SUBMIT_FH, "$submit $submit_file|");
    while(($line = readline(CONDOR_SUBMIT_FH))) {
	if($line =~ /.*submitted to cluster ([0-9]+).*/) {
	    $line =~ s/.*submitted to cluster ([0-9]+).*/\1/;
	    $job_id = $line;
	    chomp( $job_id );
	}
    }
    close(CONDOR_SUBMIT_FH);
    return $job_id;
}

sub do_remote_setup() {
	my $DAEMONS = join( ' ', @DAEMONS );
	my $glidein_condor_config;
	my $glidein_condor_config_is_temporary = 0;
	my $glidein_startup;
	my $glidein_startup_is_temporary = 0;

	print "\nRunning/verifying Glidein installation and setup...\n";
	if ( $use_config ne "" ) {
		$glidein_condor_config = $use_config;
	} else {
		$glidein_condor_config = "glidein_condor_config$suffix";
		$glidein_condor_config_is_temporary = 1;
		gen_main_config($glidein_condor_config);
	}

	if ( ! -f $glidein_condor_config ) {
		return( "accessing Condor config file: $glidein_condor_config" );
	}

	if ( $use_glidein_startup ne "" ) {
		$glidein_startup = $use_glidein_startup;
	} else {
		$glidein_startup = "glidein_startup$suffix";
		$glidein_startup_is_temporary = 1;
		gen_glidein_startup($glidein_startup);
	}

	if ( ! -f $glidein_startup ) {
		return( "accessing glidein startup file: $glidein_startup" );
	}

	my $glidein_condor_config_text;
	my $glidein_startup_text;
	my $glidein_gridmap_text = "";
	my $glidein_trusted_ca_text = "";
	my $have_gridmap = 0;
	my $have_trusted_ca = 0;

	open( CONFIG_FILE, "<$glidein_condor_config")
	    or return ("reading $glidein_condor_config: $!");
	$glidein_condor_config_text = join("",readline(CONFIG_FILE));
	close(CONFIG_FILE);
	if( $glidein_condor_config_is_temporary ) {
		unlink( $glidein_condor_config );
	}

	open( STARTUP_FILE, "<$glidein_startup")
	    or return ("reading $glidein_startup: $!");
	$glidein_startup_text = join("",readline(STARTUP_FILE));
	close(STARTUP_FILE);
	if( $glidein_startup_is_temporary ) {
		unlink( $glidein_startup );
	}

	if ( $install_gsi_gridmap ne "" ) {
	    $have_gridmap = 1;
	    open( GRIDMAP_FILE, "<$install_gsi_gridmap")
		or return ("reading $install_gsi_gridmap: $!");
	    $glidein_gridmap_text = join("",readline(GRIDMAP_FILE));
	    close(GRIDMAP_FILE);
	}
	if ( $install_gsi_trusted_ca_dir ne "" ) {
	    $have_trusted_ca = 1;
	    if ( ! -d $install_gsi_trusted_ca_dir ) {
		return "no such directory $install_gsi_trusted_ca_dir";
	    }
	    chomp($cwd = `pwd`);
	    $glidein_trusted_ca_tar = $cwd . "/glidein_trusted_ca.tar$suffix";
	    $command = "if ! cd $install_gsi_trusted_ca_dir; then exit 1; fi\n";
	    $command .= "if ! tar cf $glidein_trusted_ca_tar *; then exit 1; fi\n";
	    $command .= "if ! mimencode $glidein_trusted_ca_tar > $glidein_trusted_ca_tar.base64; then exit 1; fi\n";
	    $command .= "rm $glidein_trusted_ca_tar\n";
	    if(system($command) != 0) {
		return "building trusted ca tar file";
	    }

	    open( TRUSTED_CA_FILE, "<$glidein_trusted_ca_tar.base64")
		or return ("reading $glidein_trusted_ca_tar.base64: $!");
	    $glidein_trusted_ca_text = join("",readline(TRUSTED_CA_FILE));
	    close(TRUSTED_CA_FILE);
	    unlink( "$glidein_trusted_ca_tar.base64" );
	}

	#For maximum portability and to avoid extra problems that may
	#result from file transfers, the condor_config and startup
	#files are inserted into the remote setup script.

	#Escape $ characters.
	$glidein_condor_config_text =~ s/\$/\\\$/g;
	$glidein_startup_text =~ s/\$/\\\$/g;
	$glidein_gridmap_text =~ s/\$/\\\$/g;
	$glidein_trusted_ca_text =~ s/\$/\\\$/g;

	my $glidein_remote_setup = "glidein_remote_setup$suffix";
	open( REM_SETUP, ">$glidein_remote_setup" );
	print REM_SETUP<<"EOF";
#!/bin/sh

	BASEDIR=\$1
	ARCHDIR=\$2
	ARCH=\$3
	LOCALDIR=\$4
	TARBALL_SERVERS=\$5
	FORCE=\$6

	CONFIG=\$BASEDIR/$config_name
	STARTUP=\$ARCHDIR/$startup_name
	GRIDMAP=\$BASEDIR/grid-security/grid-mapfile
	TRUSTED_CA=\$BASEDIR/grid-security/certificates

	if [ "\$GLOBUS_SH_TAR" = "" ]; then
		#We must not have gotten executed via globus-sh-exec.
		#(In the old days, that is how this script worked.)
		#Set up the utility variables that we need.
		GLOBUS_SH_TAR=tar;
		GLOBUS_SH_GZIP=gzip;
		GLOBUS_SH_RM=rm;
		GLOBUS_SH_GREP=grep;
		GLOBUS_SH_SED=sed;
		GLOBUS_SH_MKDIR=mkdir;
		export LD_LIBRARY_PATH=\$GLOBUS_LOCATION/lib:\$LD_LIBRARY_PATH
	fi

	for newdir in \$BASEDIR \$ARCHDIR \$LOCALDIR; do
		if [ ! -d \$newdir ];
			then \$GLOBUS_SH_MKDIR \$newdir;
		fi
	if [ ! -d \$newdir ];
			then echo "ERROR mkdir \$newdir"
			exit 1;
		fi
	done

	if [ \$FORCE -ne 0 -o ! -a \$CONFIG ]; then
		#Write config file
		echo "Installing \$CONFIG.";
		cat > \$CONFIG.\$\$ <<END_CONFIG
$glidein_condor_config_text
END_CONFIG
		mv -f \$CONFIG.\$\$ \$CONFIG;
	else
		echo "Already installed: \$CONFIG.";
	fi

	if [ \$FORCE -ne 0 -o ! -x \$STARTUP ]; then
		#Write startup file
		echo "Installing \$STARTUP.";
		cat > \$STARTUP.\$\$ <<END_STARTUP
$glidein_startup_text
END_STARTUP
		chmod a+x \$STARTUP.\$\$;
		mv -f \$STARTUP.\$\$ \$STARTUP;
	else
		echo "Already installed: \$STARTUP.";
	fi

	if [ "$have_gridmap" = "0" ]; then
		#User did not supply grid-mapfile.
		echo > /dev/null  #nop
	elif [ \$FORCE -ne 0 -o ! -a \$GRIDMAP ]; then
		#Write grid-mapfile
		echo "Installing \$GRIDMAP.";
		mkdir -p `dirname \$GRIDMAP`
		cat > \$GRIDMAP.\$\$ <<END_CONFIG
$glidein_gridmap_text
END_CONFIG
		mv -f \$GRIDMAP.\$\$ \$GRIDMAP;
	else
		echo "Already installed: \$CONFIG.";
	fi

	if [ "$have_trusted_ca" = "0" ]; then
		#User did not supply trusted ca directory.
		echo > /dev/null  #nop
	elif [ \$FORCE -ne 0 -o ! -d \$TRUSTED_CA ]; then
		#Write trusted ca files
		echo "Installing \$TRUSTED_CA.";
		mkdir -p \$TRUSTED_CA
		mimencode -u > \$TRUSTED_CA.\$\$.tar <<END_CONFIG
$glidein_trusted_ca_text
END_CONFIG
		tar -x -C \$TRUSTED_CA -f \$TRUSTED_CA.\$\$.tar
	else
		echo "Already installed: \$TRUSTED_CA.";
	fi


	cd \$ARCHDIR
	GETFILES=0
	for needexec in $DAEMONS; do
		if [ ! -f \$needexec ];
			then GETFILES=1;
		fi
	done
	if [ \$FORCE -ne 0 -o \$GETFILES -ne 0 ]; then
		#Convert from comma separated URL list to space separated.
		echo "Installing Condor daemons in \$ARCHDIR.";
		TARBALL_SERVER_LIST=`echo \$TARBALL_SERVERS | \$GLOBUS_SH_SED 's/,/ /g'`;
		if [ "\$?" != "0" ]; then
			echo "Error: failed to process GLIDEIN_SERVER_URLS information.";
			exit 1;
		fi
		DOWNLOAD_SUCCESS=0
		GLOBUS_URL_COPY=\$GLOBUS_LOCATION/bin/globus-url-copy
		for TARBALL_SERVER in \$TARBALL_SERVER_LIST; do
			TARBALL_URL=\$TARBALL_SERVER/\$ARCH.tar.gz

			\$GLOBUS_URL_COPY \$TARBALL_URL file:\$ARCHDIR/\$ARCH.tar.gz;
			rc=\$?;
			if [ \$rc != 0 -o ! -f \$ARCHDIR/\$ARCH.tar.gz ] &&
			   echo \$TARBALL_URL | \$GLOBUS_SH_GREP -q -E '(^http)|(^ftp)'; then
				echo "globus-url-copy failed; trying wget"
				wget -q -O \$ARCHDIR/\$ARCH.tar.gz \$TARBALL_URL
				rc=\$?;
			fi

			if [ \$rc != 0 -o ! -f \$ARCHDIR/\$ARCH.tar.gz ]; then
				echo "Failed to retrieve \$TARBALL_URL";
			else
				if ! \$GLOBUS_SH_GZIP -d \$ARCHDIR/\$ARCH.tar.gz; then
					echo "Failed to gunzip glidein tarball."
					exit 1;
				elif ! \$GLOBUS_SH_TAR xf \$ARCHDIR/\$ARCH.tar; then
					echo "Failed to untar glidein tarball."
					exit 1;
				fi
				\$GLOBUS_SH_RM -f \$ARCHDIR/\$ARCH.tar;
				DOWNLOAD_SUCCESS=1
				echo
				echo "Downloaded \$TARBALL_URL to \$ARCHDIR."
				echo
				break;
			fi
		done
		if [ "\$DOWNLOAD_SUCCESS" != "1" ]; then
			echo "Glidein tarball not successfully downloaded from \$TARBALL_SERVERS"
			exit 1
		fi
	else
		echo "Already installed: \$ARCHDIR.";
	fi
	#NOTE: the following text is expected to verify success.
	echo "Installation successfully completed.";
	exit 0;

EOF
	close( REM_SETUP );
	chmod 0755, $glidein_remote_setup;


	$glidein_setup_submit = "glidein_setup.submit$suffix";
	$glidein_setup_output = "glidein_setup.output$suffix";
	$glidein_setup_error = "glidein_setup.error$suffix";
	$glidein_setup_log = "glidein_setup.log$suffix";


	if( $setup_here ) {
	    #Run setup script directly, rather than submitting it
	    #for remote execution on the gatekeeper.

	    my $basedir_arg = $basedir;
	    my $archdir_arg = $archdir;
	    my $localdir_arg = $localdir;
	    $basedir_arg  =~ s/\$\(HOME\)/$ENV{'HOME'}/g;
	    $archdir_arg  =~ s/\$\(HOME\)/$ENV{'HOME'}/g;
	    $localdir_arg =~ s/\$\(HOME\)/$ENV{'HOME'}/g;

	    my $setup_here_pid = open(SETUP_HERE_FH, "./$glidein_remote_setup '$basedir_arg' '$archdir_arg' '$arch' '$localdir_arg' '$SERVER_URL' '$force_setup' >& $glidein_setup_output|");

	    my $output_pos = 0;
	    while( 1 ) {
		print get_more_output($glidein_setup_output,\$output_pos);

		$child_exited = waitpid($setup_here_pid,&WNOHANG);
		if($child_exited == $setup_here_pid || $child_exited < 0) {
		    last;
		}
	    }
	    close( SETUP_HERE_FH );

	    #Show any output not already displayed.
	    print get_more_output($glidein_setup_output,\$output_pos);

	    #Check for successful completion of the setup process.
	    $err = system("grep -q 'Installation successfully completed' $glidein_setup_output");
	    if ( $err ) {
		return( "running setup script on local machine" );
	    }

	    #Cleanup temporary files.
	    unlink( $glidein_setup_output);
	    unlink( $glidein_remote_setup );

	    return( 0 );
	}


		##Escape '$' in submit file entries
	my $basedir_arg = $basedir;
	my $archdir_arg = $archdir;
	my $localdir_arg = $localdir;
	$basedir_arg =~ s/\$/\$(DOLLAR)/g;
	$archdir_arg =~ s/\$/\$(DOLLAR)/g;
	$localdir_arg =~ s/\$/\$(DOLLAR)/g;

	# We print the globus_rsl line in the submit file only if we
	# have a definite 'queue' value to tell gram
	my $globus_rsl = "";
	if ( $CONTACT_SETUP_QUEUE ne "" &&
	     $CONTACT_SETUP_QUEUE ne "unknown" ) {
	    $globus_rsl = "globus_rsl = (queue=$CONTACT_SETUP_QUEUE)";
	}

	open( SETUP_SUBMIT_FILE, ">$glidein_setup_submit" );

	print SETUP_SUBMIT_FILE <<"EOF";
	universe = Grid
	Grid_Resource = gt2 $CONTACT_SETUP
	$globus_rsl
	executable = $glidein_remote_setup

	arguments = "'$basedir_arg' '$archdir_arg' '$arch' '$localdir_arg' '$SERVER_URL' '$force_setup'"
	#avoid trouble with scratch directory creation
	remote_initialdir = /tmp
	output = $glidein_setup_output
	error = $glidein_setup_error
	log = $glidein_setup_log
	queue
EOF
	close( SETUP_SUBMIT_FILE );

	if( $gen_submit ) {
	    print "Saving glidein setup submit file: $glidein_setup_submit\n";
	    if( $use_config ) {
		print "NOTE: $use_config has been inserted into $glidein_setup_submit, so if you make further changes to the config file, you should regenerate the setup script if you plan to use it directly.\n";
	    }
	    if( $use_glidein_startup ) {
		print "NOTE: $use_glidein_startup has been inserted into $glidein_setup_submit, so if you make further changes to the startup file, you should regenerate the setup script if you plan to use it directly.\n";
	    }
	    return;
	}

	my $my_grid_proxy_err = check_grid_proxy();
	if( $my_grid_proxy_err ) {
		return $my_grid_proxy_err;
	}

	print "Submitting Glidein setup job...\n";
	my $job_id = condor_submit($glidein_setup_submit);

	if ( !$job_id ) {
		return( "running setup script on remote machine" );
	}

	#Use condor_wait to check for completion of the setup job.
	my $condor_wait_pid = open(CONDOR_WAIT_FH, "$condor_wait $glidein_setup_log|");

	my $output_pos = 0;
	my $error_pos = 0;
	my $log_pos = 0;
	while( 1 ) {
	    #Keep re-opening the output files because the files get replaced.
	    #Showing this output in realtime may help the user know if
	    #something is going wrong.
	    print get_more_output($glidein_setup_output,\$output_pos);
	    print get_more_output($glidein_setup_error,\$error_pos);

	    $child_exited = waitpid($condor_wait_pid,&WNOHANG);
	    if($child_exited == $condor_wait_pid || $child_exited < 0) {
		last;
	    }

	    #condor_wait does not pay attention to hold status,
	    #so make sure job is not on hold or we'll wait forever.
	    if(system("grep -q 'Job was held' $glidein_setup_log") == 0) {
		print STDERR "Error: the setup job has been put on hold!  Reason given:\n";
		#Sometimes it takes a few seconds for the hold reason
		#to appear in the ClassAd.
		my $hold_reason_timeout_time = time() + 30;
		while ( time() < $hold_reason_timeout_time ) {
		    if(system("condor_q -l $job_id | grep -q ^HoldReason") == 0) {
			last;
		    }
		    sleep 5;
		}
		system("condor_q -f '%s\\n' HoldReason $job_id\n");
		system("condor_rm $job_id >& /dev/null\n");
		return( "failed to run setup script on remote machine" );
	    }

	    sleep 1
	}
	close(CONDOR_WAIT_FH);

	#Show any output not already displayed.
	print get_more_output($glidein_setup_output,\$output_pos);
	print get_more_output($glidein_setup_error,\$error_pos);

	#Check for successful completion of the setup process.
	$err = system("grep -q 'Installation successfully completed' $glidein_setup_output");
	if ( $err ) {
		return( "running setup script on remote machine" );
	}

	#Cleanup temporary files.
	unlink( $glidein_setup_submit);
	unlink( $glidein_setup_output);
	unlink( $glidein_setup_error);
	unlink( $glidein_setup_log);
	unlink( $glidein_remote_setup );

	return( 0 );
}
