1 #!/usr/local/bin/perl
\r
2 ################################################################################
\r
5 # Revision: $Revision: $
\r
6 # Description: Remotely run processes on other machines
\r
7 # Author: Andrew@DeFaria.com
\r
8 # Created: Tue Jan 8 15:57:27 MST 2008
\r
12 # (c) Copyright 2008, ClearSCM, Inc., all rights reserved
\r
14 ################################################################################
\r
20 use Term::ANSIColor qw(:constants);
\r
21 use POSIX ":sys_wait_h";
\r
26 $libs = $ENV{SITE_PERLLIB} ? $ENV{SITE_PERLLIB} : "$FindBin::Bin/../lib";
\r
28 die "Unable to find libraries\n" if !$libs and !-d $libs;
\r
31 use lib "$FindBin::Bin/../lib";
\r
49 my $_totalMachines = 0;
\r
50 my $_totalExecutions = 0;
\r
51 my $_totalFailures = 0;
\r
52 my $_totalConnectFailures = 0;
\r
53 my $_totalSkips = 0;
\r
55 my (%_workerStatuses, %_workerNames);
\r
60 display "ERROR: $msg\n" if defined $msg;
\r
62 display "rexec\t[-v] [-d] [-u] <cmd>";
\r
63 display "\t-v\tTurn on verbose mode";
\r
64 display "\t-d\tTurn on debug mode";
\r
65 display "\t-u\tThis usage message";
\r
66 display "<cmd>\tCommand to execute remotely";
\r
72 display YELLOW . "Machines: " . RESET . "$_totalMachines " .
\r
73 MAGENTA . "Executions/Failures: " . RESET . "($_totalExecutions/$_totalFailures) " .
\r
74 BLUE . "Connect Failures/Skips: " . RESET . "($_totalConnectFailures/$_totalSkips)";
\r
80 display BLUE . "\nInterrupted execution on $_host" . RESET;
\r
84 display_nolf "Executing on " . YELLOW . $_host . RESET . " - "
\r
85 . GREEN . BOLD . "S" . RESET . GREEN . "kip" . RESET . ", "
\r
86 . CYAN . BOLD . "C" . RESET . CYAN . "ontinue" . RESET . " or "
\r
87 . MAGENTA . BOLD . "A" . RESET . MAGENTA . "bort run" . RESET . " ("
\r
88 . GREEN . BOLD . "s" . RESET . "/"
\r
89 . CYAN . BOLD . "C" . RESET . "/"
\r
90 . MAGENTA . BOLD . "a" . RESET . ")?";
\r
92 ReadMode ("cbreak");
\r
93 my $answer = ReadKey (0);
\r
94 ReadMode ("normal");
\r
96 if ($answer eq "\n") {
\r
102 $answer = lc $answer;
\r
104 if ($answer eq "s") {
\r
106 display "Skipping $_host";
\r
109 } elsif ($answer eq "a") {
\r
110 display RED . "Aborting run". RESET;
\r
114 display "Continuing...";
\r
120 while ((my $worker = waitpid (-1, WNOHANG)) > 0) {
\r
123 # Ignore all child deaths except for processes we started
\r
124 next if !exists $_workerStatuses{$worker};
\r
126 $_workerStatuses{$worker} = $status;
\r
129 $SIG{CHLD} = \&workerDeath;
\r
132 sub execute ($$$) {
\r
133 my ($cmd, $host, $prompt) = @_;
\r
137 verbose_nolf "Connecting to machine $host...";
\r
140 $_currentHost = new Rexec (
\r
146 # Problem with creating Rexec object. Log error if logging and return.
\r
147 if ($@ or !$_currentHost) {
\r
149 my $log = new Logger (name => $_host);
\r
151 $log->err ("Unable to connect to $host to execute command\n$cmd");
\r
154 $_totalConnectFailures++;
\r
159 verbose " connected";
\r
161 display YELLOW . "$host:" . RESET . UNDERLINE . "$cmd" . RESET unless $_quiet;
\r
163 @lines = $_currentHost->exec ($cmd);
\r
166 # Kick current connection
\r
167 kill INT => $_currentHost->{handle}->pid;
\r
170 if ($_parallel != 0) {
\r
172 my $log = new Logger (name => $_host);
\r
174 $log->err ("Unable to connect to $host to execute command\n$cmd");
\r
177 $_totalConnectFailures++;
\r
180 verbose "Disconnected from $host";
\r
182 my $status = $_currentHost->status;
\r
184 undef $_currentHost;
\r
186 return ($status, @lines);
\r
189 sub parallelize ($%) {
\r
190 my ($cmd, %machines) = @_;
\r
192 my $thread_count = 1;
\r
194 foreach $_host (sort keys %machines) {
\r
195 if ($thread_count <= $_parallel) {
\r
196 debug "Processing $_host ($thread_count)";
\r
199 if (my $pid = fork) {
\r
200 # In parent process - record this host and its status
\r
201 $_workerNames{$pid} = $_host;
\r
203 # In spawned child...
\r
206 debug "Starting process for $_host [$pid]";
\r
208 $_workerNames{$pid} = $_host;
\r
210 my ($status, @lines) = execute $cmd, $_host, $machines{$_host};
\r
212 my $log = new Logger (name => $_host);
\r
214 $log->log ($_) foreach (@lines);
\r
219 # Wait for somebody to finish;
\r
220 debug "Waiting for somebody to exit...";
\r
223 debug "Reaped $_workerNames{$reaped} [$reaped] (Status: $?)";
\r
224 $_workerStatuses{$reaped} = $? >> 8 if $reaped != -1;
\r
230 # Wait for all kids
\r
231 my %threads = %_workerNames;
\r
233 foreach (keys %threads) {
\r
234 if (waitpid ($_, 0) == -1) {
\r
235 delete $threads{$_};
\r
237 $_workerStatuses{$_} = $? >> 8;
\r
238 debug "$threads{$_} [$_] exited with a status of $_workerStatuses{$_}";
\r
242 debug "All processed completed - Status:";
\r
245 foreach (sort keys %_workerStatuses) {
\r
246 debug "$_workerNames{$_}\t[$_]:\tStatus: $_workerStatuses{$_}";
\r
251 display "Output of all executions";
\r
252 foreach $_host (sort keys %machines) {
\r
253 if (-f "$_host.log") {
\r
254 display "$_host:$_" foreach (ReadFile ("$_host.log"));
\r
256 #unlink "$_host.log";
\r
258 warning "Unable to find output for $_host ($_host.log missing)";
\r
263 # Print the totals if interrupted
\r
264 $SIG{INT} = \&Interrupted;
\r
268 "usage" => sub { Usage "" },
\r
269 "verbose" => sub { set_verbose },
\r
270 "debug" => sub { set_debug },
\r
272 "quiet" => \$_quiet,
\r
273 "file=s" => \$_alternateFile,
\r
274 "parallel:i" => \$_parallel,
\r
275 ) || Usage "Unknown parameter";
\r
277 my $cmd = join " ", @ARGV;
\r
279 error "No command specified", 1 if !$cmd;
\r
281 my $machines = Machines->new (file => $_alternateFile);
\r
282 my %machines = $machines->all ();
\r
284 if ($_parallel > 0) {
\r
285 parallelize ($cmd, %machines);
\r
290 display "NOTE: Logging output to <host>.log" if $_log;
\r
292 foreach $_host (sort keys (%machines)) {
\r
295 my ($status, @lines) = execute $cmd, $_host, $machines{$_host};
\r
302 if (defined $status) {
\r
303 if ($status == 0) {
\r
304 $_totalExecutions++;
\r
307 my $log = new Logger (name => $_host);
\r
309 $log->log ("Host: $_host\nCommand: $cmd\nStatus: $status\nOutput:\n");
\r
310 $log->log ($_) foreach (@lines);
\r
320 my $log = new Logger (name => $_host);
\r
322 $log->log ("Host: $_host\nCommand: $cmd\nStatus: $status\nOutput:\n");
\r
323 $log->log ($_) foreach (@lines);
\r
325 display $_ foreach (@lines);
\r