2 ################################################################################
4 # File: $RCSfile: rexec,v $
5 # Revision: $Revision: 1.1 $
6 # Description: Remotely run processes on other machines
7 # Author: Andrew@ClearSCM.com
8 # Created: Tue Jan 8 15:57:27 MST 2008
9 # Modified: $Date: 2008/02/29 15:09:15 $
12 # (c) Copyright 2008, ClearSCM, Inc., all rights reserved
14 ################################################################################
20 use Term::ANSIColor qw(:constants);
21 use POSIX ":sys_wait_h";
26 $libs = $ENV{SITE_PERLLIB} ? $ENV{SITE_PERLLIB} : "$FindBin::Bin/lib";
28 die "Unable to find libraries\n" if !$libs and !-d $libs;
48 my $_totalMachines = 0;
49 my $_totalExecutions = 0;
50 my $_totalFailures = 0;
51 my $_totalConnectFailures = 0;
54 my (%_workerStatuses, %_workerNames);
59 display "ERROR: $msg\n" if defined $msg;
61 display "rexec\t[-v] [-d] [-u] <cmd>";
62 display "\t-v\tTurn on verbose mode";
63 display "\t-d\tTurn on debug mode";
64 display "\t-u\tThis usage message";
65 display "<cmd>\tCommand to execute remotely";
71 display YELLOW . "Machines: " . RESET . "$_totalMachines " .
72 MAGENTA . "Executions/Failures: " . RESET . "($_totalExecutions/$_totalFailures) " .
73 BLUE . "Connect Failures/Skips: " . RESET . "($_totalConnectFailures/$_totalSkips)";
79 display BLUE . "\nInterrupted execution on $_host" . RESET;
83 display_nolf "Executing on " . YELLOW . $_host . RESET . " - "
84 . GREEN . BOLD . "S" . RESET . GREEN . "kip" . RESET . ", "
85 . CYAN . BOLD . "C" . RESET . CYAN . "ontinue" . RESET . " or "
86 . MAGENTA . BOLD . "A" . RESET . MAGENTA . "bort run" . RESET . " ("
87 . GREEN . BOLD . "s" . RESET . "/"
88 . CYAN . BOLD . "C" . RESET . "/"
89 . MAGENTA . BOLD . "a" . RESET . ")?";
92 my $answer = ReadKey (0);
95 if ($answer eq "\n") {
101 $answer = lc $answer;
103 if ($answer eq "s") {
105 display "Skipping $_host";
108 } elsif ($answer eq "a") {
109 display RED . "Aborting run". RESET;
113 display "Continuing...";
119 while ((my $worker = waitpid (-1, WNOHANG)) > 0) {
122 # Ignore all child deaths except for processes we started
123 next if !exists $_workerStatuses{$worker};
125 $_workerStatuses{$worker} = $status;
128 $SIG{CHLD} = \&workerDeath;
132 my ($cmd, $host, $prompt) = @_;
136 verbose_nolf "Connecting to machine $host...";
139 $_currentHost = new Rexec (
145 # Problem with creating Rexec object. Log error if logging and return.
146 if ($@ or !$_currentHost) {
148 my $log = new Logger (name => $_host);
150 $log->err ("Unable to connect to $host to execute command\n$cmd");
153 $_totalConnectFailures++;
158 verbose " connected";
160 display YELLOW . "$host:" . RESET . UNDERLINE . "$cmd" . RESET unless $_quiet;
162 @lines = $_currentHost->exec ($cmd);
165 # Kick current connection
166 kill INT => $_currentHost->{handle}->pid;
169 if ($_parallel != 0) {
171 my $log = new Logger (name => $_host);
173 $log->err ("Unable to connect to $host to execute command\n$cmd");
176 $_totalConnectFailures++;
179 verbose "Disconnected from $host";
181 my $status = $_currentHost->status;
185 return ($status, @lines);
188 sub parallelize ($%) {
189 my ($cmd, %machines) = @_;
191 my $thread_count = 1;
193 foreach $_host (sort keys %machines) {
194 if ($thread_count <= $_parallel) {
195 debug "Processing $_host ($thread_count)";
198 if (my $pid = fork) {
199 # In parent process - record this host and its status
200 $_workerNames{$pid} = $_host;
202 # In spawned child...
205 debug "Starting process for $_host [$pid]";
207 $_workerNames{$pid} = $_host;
209 my ($status, @lines) = execute $cmd, $_host, $machines{$_host};
211 my $log = new Logger (name => $_host);
213 $log->log ($_) foreach (@lines);
218 # Wait for somebody to finish;
219 debug "Waiting for somebody to exit...";
222 debug "Reaped $_workerNames{$reaped} [$reaped] (Status: $?)";
223 $_workerStatuses{$reaped} = $? >> 8 if $reaped != -1;
230 my %threads = %_workerNames;
232 foreach (keys %threads) {
233 if (waitpid ($_, 0) == -1) {
236 $_workerStatuses{$_} = $? >> 8;
237 debug "$threads{$_} [$_] exited with a status of $_workerStatuses{$_}";
241 debug "All processed completed - Status:";
244 foreach (sort keys %_workerStatuses) {
245 debug "$_workerNames{$_}\t[$_]:\tStatus: $_workerStatuses{$_}";
250 display "Output of all executions";
251 foreach $_host (sort keys %machines) {
252 if (-f "$_host.log") {
253 display "$_host:$_" foreach (ReadFile ("$_host.log"));
255 #unlink "$_host.log";
257 warning "Unable to find output for $_host ($_host.log missing)";
262 # Print the totals if interrupted
263 $SIG{INT} = \&Interrupted;
267 "usage" => sub { Usage "" },
268 "verbose" => sub { set_verbose },
269 "debug" => sub { set_debug },
272 "file=s" => \$_alternateFile,
273 "parallel:i" => \$_parallel,
274 ) || Usage "Unknown parameter";
276 my $cmd = join " ", @ARGV;
278 error "No command specified", 1 if !$cmd;
280 my $machines = Machines->new (file => $_alternateFile);
281 my %machines = $machines->all ();
283 if ($_parallel > 0) {
284 parallelize ($cmd, %machines);
289 display "NOTE: Logging output to <host>.log" if $_log;
291 foreach $_host (sort keys (%machines)) {
294 my ($status, @lines) = execute $cmd, $_host, $machines{$_host};
301 if (defined $status) {
306 my $log = new Logger (name => $_host);
308 $log->err ("Unable to execute command on $_host\n$cmd");
318 my $log = new Logger (name => $_host);
320 $log->log ($_) foreach (@lines);
322 display $_ foreach (@lines);