+#!/usr/local/bin/perl\r
+################################################################################\r
+#\r
+# File: $RCSfile: $\r
+# Revision: $Revision: $\r
+# Description: Remotely run processes on other machines\r
+# Author: Andrew@DeFaria.com\r
+# Created: Tue Jan 8 15:57:27 MST 2008\r
+# Modified: $Date: $\r
+# Language: perl\r
+#\r
+# (c) Copyright 2008, ClearSCM, Inc., all rights reserved\r
+#\r
+################################################################################\r
+use strict;\r
+use warnings;\r
+\r
+use FindBin;\r
+use Getopt::Long;\r
+use Term::ANSIColor qw(:constants);\r
+use POSIX ":sys_wait_h";\r
+\r
+my $libs;\r
+\r
+BEGIN {\r
+ $libs = $ENV{SITE_PERLLIB} ? $ENV{SITE_PERLLIB} : "$FindBin::Bin/../lib";\r
+\r
+ die "Unable to find libraries\n" if !$libs and !-d $libs;\r
+}\r
+\r
+use lib "$FindBin::Bin/../lib";\r
+use lib $libs;\r
+\r
+use Display;\r
+use Logger;\r
+use Machines;\r
+use Rexec;\r
+use Utils;\r
+\r
+our $_host;\r
+our $_skip = 0;\r
+our $_currentHost;\r
+\r
+my $_log = 0;\r
+my $_quiet = 0;\r
+my $_alternateFile;\r
+my $_parallel = 0;\r
+\r
+my $_totalMachines = 0;\r
+my $_totalExecutions = 0;\r
+my $_totalFailures = 0;\r
+my $_totalConnectFailures = 0;\r
+my $_totalSkips = 0;\r
+\r
+my (%_workerStatuses, %_workerNames);\r
+\r
+sub Usage {\r
+ my $msg = shift;\r
+\r
+ display "ERROR: $msg\n" if defined $msg;\r
+\r
+ display "rexec\t[-v] [-d] [-u] <cmd>";\r
+ display "\t-v\tTurn on verbose mode";\r
+ display "\t-d\tTurn on debug mode";\r
+ display "\t-u\tThis usage message";\r
+ display "<cmd>\tCommand to execute remotely";\r
+\r
+ exit 1;\r
+} # Usage\r
+\r
+sub printStats {\r
+ display YELLOW . "Machines: " . RESET . "$_totalMachines " .\r
+ MAGENTA . "Executions/Failures: " . RESET . "($_totalExecutions/$_totalFailures) " .\r
+ BLUE . "Connect Failures/Skips: " . RESET . "($_totalConnectFailures/$_totalSkips)";\r
+} # printStats\r
+\r
+sub Interrupted {\r
+ use Term::ReadKey;\r
+\r
+ display BLUE . "\nInterrupted execution on $_host" . RESET;\r
+\r
+ printStats;\r
+\r
+ display_nolf "Executing on " . YELLOW . $_host . RESET . " - "\r
+ . GREEN . BOLD . "S" . RESET . GREEN . "kip" . RESET . ", "\r
+ . CYAN . BOLD . "C" . RESET . CYAN . "ontinue" . RESET . " or "\r
+ . MAGENTA . BOLD . "A" . RESET . MAGENTA . "bort run" . RESET . " ("\r
+ . GREEN . BOLD . "s" . RESET . "/"\r
+ . CYAN . BOLD . "C" . RESET . "/"\r
+ . MAGENTA . BOLD . "a" . RESET . ")?";\r
+\r
+ ReadMode ("cbreak");\r
+ my $answer = ReadKey (0);\r
+ ReadMode ("normal");\r
+\r
+ if ($answer eq "\n") {\r
+ display "c";\r
+ } else {\r
+ display $answer;\r
+ } # if\r
+\r
+ $answer = lc $answer;\r
+\r
+ if ($answer eq "s") {\r
+ *STDOUT->flush;\r
+ display "Skipping $_host";\r
+ $_skip = 1;\r
+ $_totalSkips++;\r
+ } elsif ($answer eq "a") {\r
+ display RED . "Aborting run". RESET;\r
+ printStats;\r
+ exit;\r
+ } else {\r
+ display "Continuing...";\r
+ $_skip = 0;\r
+ } # if\r
+} # Interrupted\r
+\r
+sub workerDeath {\r
+ while ((my $worker = waitpid (-1, WNOHANG)) > 0) {\r
+ my $status = $?;\r
+\r
+ # Ignore all child deaths except for processes we started\r
+ next if !exists $_workerStatuses{$worker};\r
+\r
+ $_workerStatuses{$worker} = $status;\r
+ } # while\r
+\r
+ $SIG{CHLD} = \&workerDeath;\r
+} # workerDeath\r
+\r
+sub execute ($$$) {\r
+ my ($cmd, $host, $prompt) = @_;\r
+\r
+ my @lines;\r
+\r
+ verbose_nolf "Connecting to machine $host...";\r
+\r
+ eval {\r
+ $_currentHost = new Rexec (\r
+ host => $host,\r
+ prompt => $prompt,\r
+ );\r
+ };\r
+\r
+ # Problem with creating Rexec object. Log error if logging and return.\r
+ if ($@ or !$_currentHost) {\r
+ if ($_log) {\r
+ my $log = new Logger (name => $_host);\r
+\r
+ $log->err ("Unable to connect to $host to execute command\n$cmd");\r
+ } # if\r
+\r
+ $_totalConnectFailures++;\r
+\r
+ return (1, ());\r
+ } # if\r
+\r
+ verbose " connected";\r
+\r
+ display YELLOW . "$host:" . RESET . UNDERLINE . "$cmd" . RESET unless $_quiet;\r
+\r
+ @lines = $_currentHost->exec ($cmd);\r
+\r
+ if ($_skip) {\r
+ # Kick current connection\r
+ kill INT => $_currentHost->{handle}->pid;\r
+ } # if\r
+\r
+ if ($_parallel != 0) {\r
+ if ($_log) {\r
+ my $log = new Logger (name => $_host);\r
+\r
+ $log->err ("Unable to connect to $host to execute command\n$cmd");\r
+ } # if\r
+\r
+ $_totalConnectFailures++;\r
+ } # if\r
+\r
+ verbose "Disconnected from $host";\r
+\r
+ my $status = $_currentHost->status;\r
+\r
+ undef $_currentHost;\r
+\r
+ return ($status, @lines);\r
+} # execute\r
+\r
+sub parallelize ($%) {\r
+ my ($cmd, %machines) = @_;\r
+\r
+ my $thread_count = 1;\r
+\r
+ foreach $_host (sort keys %machines) {\r
+ if ($thread_count <= $_parallel) {\r
+ debug "Processing $_host ($thread_count)";\r
+ $thread_count++;\r
+\r
+ if (my $pid = fork) {\r
+ # In parent process - record this host and its status\r
+ $_workerNames{$pid} = $_host;\r
+ } else {\r
+ # In spawned child...\r
+ $pid = $$;\r
+\r
+ debug "Starting process for $_host [$pid]";\r
+\r
+ $_workerNames{$pid} = $_host;\r
+ \r
+ my ($status, @lines) = execute $cmd, $_host, $machines{$_host};\r
+\r
+ my $log = new Logger (name => $_host);\r
+\r
+ $log->log ($_) foreach (@lines);\r
+\r
+ exit $status;\r
+ } # if\r
+ } else {\r
+ # Wait for somebody to finish;\r
+ debug "Waiting for somebody to exit...";\r
+ my $reaped = wait;\r
+\r
+ debug "Reaped $_workerNames{$reaped} [$reaped] (Status: $?)";\r
+ $_workerStatuses{$reaped} = $? >> 8 if $reaped != -1;\r
+\r
+ $thread_count--;\r
+ } # if\r
+ } # foreach\r
+\r
+ # Wait for all kids\r
+ my %threads = %_workerNames;\r
+\r
+ foreach (keys %threads) {\r
+ if (waitpid ($_, 0) == -1) {\r
+ delete $threads{$_};\r
+ } else {\r
+ $_workerStatuses{$_} = $? >> 8;\r
+ debug "$threads{$_} [$_] exited with a status of $_workerStatuses{$_}";\r
+ } # if\r
+ } # foreach\r
+\r
+ debug "All processed completed - Status:";\r
+\r
+ if (get_debug) {\r
+ foreach (sort keys %_workerStatuses) {\r
+ debug "$_workerNames{$_}\t[$_]:\tStatus: $_workerStatuses{$_}";\r
+ } # foreach\r
+ } # if\r
+\r
+ # Gather output...\r
+ display "Output of all executions";\r
+ foreach $_host (sort keys %machines) {\r
+ if (-f "$_host.log") {\r
+ display "$_host:$_" foreach (ReadFile ("$_host.log"));\r
+\r
+ #unlink "$_host.log";\r
+ } else {\r
+ warning "Unable to find output for $_host ($_host.log missing)";\r
+ } # if\r
+ } # foreach\r
+} # parallelize\r
+\r
+# Print the totals if interrupted\r
+$SIG{INT} = \&Interrupted;\r
+\r
+# Get our options\r
+GetOptions (\r
+ "usage" => sub { Usage "" },\r
+ "verbose" => sub { set_verbose },\r
+ "debug" => sub { set_debug },\r
+ "log" => \$_log,\r
+ "quiet" => \$_quiet,\r
+ "file=s" => \$_alternateFile,\r
+ "parallel:i" => \$_parallel,\r
+) || Usage "Unknown parameter";\r
+\r
+my $cmd = join " ", @ARGV;\r
+\r
+error "No command specified", 1 if !$cmd;\r
+\r
+my $machines = Machines->new (file => $_alternateFile);\r
+my %machines = $machines->all ();\r
+\r
+if ($_parallel > 0) {\r
+ parallelize ($cmd, %machines);\r
+ printStats;\r
+ exit;\r
+} # if\r
+\r
+display "NOTE: Logging output to <host>.log" if $_log;\r
+\r
+foreach $_host (sort keys (%machines)) {\r
+ $_totalMachines++;\r
+\r
+ my ($status, @lines) = execute $cmd, $_host, $machines{$_host};\r
+\r
+ if ($_skip) {\r
+ $_skip = 0;\r
+ next;\r
+ } # if\r
+\r
+ if (defined $status) {\r
+ if ($status == 0) {\r
+ $_totalExecutions++;\r
+ } else {\r
+ if ($_log) {\r
+ my $log = new Logger (name => $_host);\r
+\r
+ $log->log ("Host: $_host\nCommand: $cmd\nStatus: $status\nOutput:\n");\r
+ $log->log ($_) foreach (@lines);\r
+ } # if\r
+ \r
+ $_totalFailures++;\r
+\r
+ next;\r
+ } # if\r
+ } # if\r
+\r
+ if ($_log) {\r
+ my $log = new Logger (name => $_host);\r
+\r
+ $log->log ("Host: $_host\nCommand: $cmd\nStatus: $status\nOutput:\n");\r
+ $log->log ($_) foreach (@lines);\r
+ } else {\r
+ display $_ foreach (@lines);\r
+ } # if\r
+} # foreach\r
+\r
+printStats;\r