5 =head1 NAME $RCSfile: cleartasks.pl,v $
15 Andrew DeFaria <Andrew@ClearSCM.com>
23 Sun Jan 2 19:40:28 EST 2011
27 $Date: 2013/06/02 18:47:26 $
33 Usage cleartasks.pl: [-u|sage] [-ve|rbose] [-deb|ug]
36 -u|sage: Displays usage
39 -de|bug: Output debug messages
41 -da|emon: Run in daemon mode (Default: yes)
42 -p|idfile: File to be created with the pid written to it (Default:
43 cleartasks.pid). Note: pidfile is only written if -daemon is
48 Examine the Clearadm schedule and perform the tasks required.
50 Note that sending the Cleartasks.pl process a sigusr1 will cause it to toggle
62 use lib "$FindBin::Bin/lib", "$FindBin::Bin/../lib";
71 my $VERSION = '$Revision: 1.25 $';
72 ($VERSION) = ($VERSION =~ /\$Revision: (.*) /);
74 my $logfile = "$Clearadm::CLEAROPTS{CLEARADM_LOGDIR}/$FindBin::Script";
75 $logfile =~ s/\.pl$//;
76 $logfile .= '.' . hostname() . '.log';
78 my $pidfile = "$Clearadm::CLEAROPTS{CLEARADM_RUNDIR}/$FindBin::Script.pid";
81 # Augment PATH with $Clearadm::CLEAROPTS{CLEARADM_BASE}
82 $ENV{PATH} .= ":$Clearadm::CLEAROPTS{CLEARADM_BASE}";
84 my ($clearadm, $clearexec);
88 display 'Turning verbose off';
91 display 'Turning verbose on';
96 $SIG{USR1} = \&ToggleVerbose;
98 sub HandleSystemNotCheckingIn (%) {
101 my $startTime = time;
103 my $message = "Unable to connect to system $system{name}:$system{port}";
106 task => 'System checkin',
107 started => Today2SQLDatetime,
110 system => $system{name},
113 my ($err, $msg, $lastid) = $clearadm->AddRunlog (%runlog);
115 $clearadm->Error ("Unable to add to runlog (Status: $err)\n$msg") if $err;
117 # Check to see if we should notify anybody about this non-responding system
118 my %notification = $clearadm->GetNotification ('System checkin');
120 my $when = Today2SQLDatetime;
121 my $nomorethan = lc $notification{nomorethan};
122 my $systemLink = $Clearadm::CLEAROPTS{CLEARADM_WEBBASE};
123 $systemLink .= "/systemdetails.cgi?system=$system{name}";
124 my $runlogLink = $Clearadm::CLEAROPTS{CLEARADM_WEBBASE};
125 $runlogLink .= "/runlog.cgi?id=$lastid";
126 my $subject = "System is not responding (Is clearagent running?)";
129 <h1><font color="red">Alert</font> System not responding!</h1>
132 <p>On $when the system <a href="$systemLink">$system{name}</a> was <a
133 href="$runlogLink">not responding</a> to clearagent requests. This can happen if
134 clearagent is not setup and running on the system.</p>
147 verbose "$system{name}: $subject";
150 } # HandleSystemNotCheckingIn
152 sub SystemsCheckin () {
153 for ($clearadm->FindSystem) {
156 next if $system{active} eq 'false';
158 verbose "Contacting system $system{name}:$system{port}";
160 my $startTime = time;
162 my $status = $clearexec->connectToServer (
168 HandleSystemNotCheckingIn %system;
172 $clearexec->disconnectFromServer;
174 verbose 'Successfully checked in with system: '
175 . "$system{name}:$system{port}";
177 display __FILE__ . " DEBUG: System undefined 1" unless $system{name};
178 $clearadm->UpdateSystem (
180 (lastheardfrom => Today2SQLDatetime)
183 $clearadm->ClearNotifications ($system{name})
184 if $system{notification} and $system{notification} eq 'Heartbeat';
190 sub UpdateRunlog ($$$$) {
191 my ($status, $startTime, $task, $output) = @_;
194 task => $$task{name},
195 system => $$task{system},
196 started => Today2SQLDatetime,
199 $runlog{status} = $status;
203 $runlog{message} = join "\n", @$output;
205 $runlog{message} = 'Successful execution of ';
206 $runlog{message} .= "$$task{name}: $$task{command}";
210 $runlog{message} = join "\n", @$output;
212 $runlog{message} = 'Unable to execute ';
213 $runlog{message} .= "$$task{name}: $$task{command} ";
214 $runlog{message} .= join (' ', @$output);
218 my ($err, $msg, $lastid) = $clearadm->AddRunlog (%runlog);
220 $clearadm->Error ($msg, $err) if $err;
225 sub MakeSystemLink ($) {
228 return "$Clearadm::CLEAROPTS{CLEARADM_WEBBASE}/systemdetails.cgi?system="
232 sub MakeLoadavgLink ($) {
235 return "$Clearadm::CLEAROPTS{CLEARADM_WEBBASE}/plot.cgi?type=loadavg&system="
236 . "$system&scaling=Hour&points=24";
239 sub ProcessLoadavgErrors ($$$$@) {
240 # TODO: Also need to handle the case where the error was something other
241 # than "Load average over threshold". Perhaps by having different return
242 # status. Also, runlog entry #22169 never reported!
243 my ($notification, $task, $system, $lastid, @output) = @_;
245 my $when = Today2SQLDatetime;
248 # We need to log this output. Write it to STDOUT
251 my ($subject, $message, $currLoadavg, $threshold, $systemLink, $loadavgLink);
253 if (/System: (\w+) Loadavg (\d+\.\d+) Threshold (\d+\.\d+)/) {
257 $systemLink = MakeSystemLink $system;
258 $loadavgLink = MakeLoadavgLink $system;
259 $subject = "Load average of $currLoadavg exceeds threshold ";
260 $subject .= "($threshold)";
263 <h1><font color="red">Alert</font> Load Average is over the threshold!</h1>
266 <p>On $when the system <a href="$systemLink">$system</a>'s load avg
267 (<a href="$loadavgLink">$currLoadavg</a>) had exceeded the threshold set for
268 this system ($threshold).</p>
270 } elsif (/ERROR.*system\s+(\S+):/) {
272 $systemLink = MakeSystemLink $system;
273 $subject = "Error trying to obtain Loadavg";
276 <h1><font color="red">Alert</font> Unable to obtain Loadavg!</h1>
279 <p>On $when we were unable to obtain the Loadavg for
280 system <a href="$systemLink">$system</a>.</p>
282 <p>The following was the error message:</p>
287 <p>On $when on the system $system, we were unable to parse the Loadavg output. This is what we saw:</p>
291 $message .= join "\n", @output;
292 $message .= "</pre>";
293 $clearadm->Error ($message, -1);
310 } # ProcessLoadAvgErrors
312 sub ProcessFilesystemErrors ($$$$@) {
313 # TODO: Also need to handle the case where the error was something other
314 # than "Filesystem over threshold". Perhaps by having different return
316 my ($notification, $task, $system, $lastid, @output) = @_;
318 my $when = Today2SQLDatetime;
323 # We need to log this output. Write it to STDOUT
326 if (/System:\s*(\S+)\s*Filesystem:\s*(\S+)\s*Used:\s*(\d+\.\d+)%\s*Threshold:\s*(\d+)/) {
334 $system{$1} = [$system{$1}, \%fsinfo];
336 $system{$1} = \%fsinfo;
341 for my $systemName (keys %system) {
344 if (ref $system{$systemName} eq 'HASH') {
345 push @fsinfo, $system{$systemName};
347 push @fsinfo, @{$system{$systemName}};
350 my $systemLink = MakeSystemLink ($systemName);
351 my $subject = 'Filesystem has exceeded threshold';
352 my $message = <<"END";
354 <h1><font color="red">Alert</font> Filesystem is over the threshold!</h1>
357 <p>On $when the following filesystems on <a href="$systemLink">$systemName</a>
358 were over their threshold.</p>
364 my $filesystemLink = $Clearadm::CLEAROPTS{CLEARADM_WEBBASE};
365 $filesystemLink .= "/plot.cgi?type=filesystem&system=$systemName";
366 $filesystemLink .= "&filesystem=$fsinfo{filesystem}";
367 $filesystemLink .= '&scaling=Day&points=7';
368 $message .= "<li>Filesystem <a href=\"$filesystemLink\">";
369 $message .= "$fsinfo{filesystem}</a> is $fsinfo{usedPct}% full. Threshold is ";
370 $message .= "$fsinfo{threshold}%</li>";
387 } # ProcessFilesystemErrors
389 sub NonZeroReturn ($$$$$$) {
390 my ($system, $notification, $status, $lastid, $output, $task) = @_;
392 my @output = @{$output};
395 my $when = Today2SQLDatetime;
397 my $subject = "Non zero return from $task{command} "
398 . "executing on $system";
399 my $taskLink = $Clearadm::CLEAROPTS{CLEARADM_WEBBASE};
400 $taskLink .= "/tasks.cgi?task=$task{name}";
401 my $similarLink = $Clearadm::CLEAROPTS{CLEARADM_WEBBASE};
402 $similarLink .= "/runlog.cgi?system=$task{system}"
404 . "&task=$task{name}";
405 my $runlogLink = $Clearadm::CLEAROPTS{CLEARADM_WEBBASE};
406 $runlogLink .= "/runlog.cgi?id=$lastid";
407 my $message = <<"END";
409 <h1><font color="red">Alert</font> Non zero status from script execution!</h1>
412 <p>On $when, while executing <a href="$taskLink">$task{name}</a> on
413 $task{system}, a non zero status of $status was returned. Here is the resulting
414 output:</p><blockquote><pre>
417 $message .= join "\n", @output;
420 <p>You may wish to examine the individual <a href="$runlogLink">runlog entry</a>
421 that caused this alert or a list of <a href="$similarLink">similar
425 $message .= "</pre></blockquote>";
440 sub ExecuteTask ($%) {
441 my ($sleep, %task) = @_;
443 my ($status, @output, %system, $subject, $message);
445 verbose_nolf "Performing task $task{name}";
447 my %notification = $clearadm->GetNotification ($task{notification});
449 my $startTime = time;
451 if ($task{system} =~ /localhost/i) {
452 verbose " on localhost";
453 ($status, @output) = Execute "$task{command} 2>&1";
455 %system = $clearadm->GetSystem ($task{system});
457 verbose " on $system{name}";
459 $status = $clearexec->connectToServer (
465 $output[0] = "Unable to connect to system $system{name}:$system{port} to "
466 . "execute $task{command}";
469 ($status, @output) = $clearexec->execute ($task{command});
471 $output[0] = "Unable to exec $task{command} on $system{name}"
475 $clearexec->disconnectFromServer;
478 my $lastid = UpdateRunlog ($status, $startTime, \%task, \@output);
481 if ($notification{cond}
482 and $notification{cond} =~ /non zero return/i) {
491 } elsif ($notification{cond} =~ /loadavg over threshold/i) {
492 ProcessLoadavgErrors ($notification{name}, $task{name}, $system{name}, $lastid, @output);
493 } elsif ($notification{cond} =~ /filesystem over threshold/i) {
494 ProcessFilesystemErrors ($notification{name}, $task{name}, $system{name}, $lastid, @output);
497 $clearadm->ClearNotifications ($task{system});
500 my ($err, $msg) = $clearadm->UpdateSchedule (
502 ( 'lastrunid' => $lastid ),
505 $clearadm->Error ($msg, $err) if $err;
507 $sleep -= time - $startTime;
514 'usage' => sub { Usage },
515 'verbose' => sub { set_verbose },
516 'debug' => sub { set_debug },
517 'daemon!' => \$daemon,
518 'pidfile=s' => \$pidfile,
519 ) or Usage "Invalid parameter";
521 Usage 'Extraneous options: ' . join ' ', @ARGV
524 EnterDaemonMode $logfile, $logfile, $pidfile
527 display "$FindBin::Script V$VERSION started at " . localtime;
529 $clearadm = Clearadm->new;
530 $clearexec = Clearexec->new;
532 $clearadm->SetNotify;
535 # First check in with all systems
538 my ($sleep, @workItems) = $clearadm->GetWork;
541 my %scheduledTask = %{$_};
543 $scheduledTask{system} ||= 'All systems';
545 if ($scheduledTask{system} =~ /all systems/i) {
546 for my $system ($clearadm->FindSystem) {
547 next if $$system{active} eq 'false';
549 $scheduledTask{system} = $$system{name};
550 $sleep = ExecuteTask $sleep, %scheduledTask;
553 $sleep = ExecuteTask $sleep, %scheduledTask;
558 verbose "Sleeping for $sleep seconds";
565 =head1 CONFIGURATION AND ENVIRONMENT
567 DEBUG: If set then $debug is set to this level.
569 VERBOSE: If set then $verbose is set to this level.
571 TRACE: If set then $trace is set to this level.
579 L<Getopt::Long|Getopt::Long>
581 =head2 ClearSCM Perl Modules
597 <a href="http://clearscm.com/php/scm_man.php?file=clearadm/lib/Clearadm.pm">Clearadm</a><br>
598 <a href="http://clearscm.com/php/scm_man.php?file=clearadm/lib/Clearexec.pm">Clearexec</a><br>
599 <a href="http://clearscm.com/php/scm_man.php?file=lib/DateUtils.pm">DateUtils</a><br>
600 <a href="http://clearscm.com/php/scm_man.php?file=lib/Display.pm">Display</a><br>
601 <a href="http://clearscm.com/php/scm_man.php?file=lib/TimeUtils.pm">TimeUtils</a><br>
602 <a href="http://clearscm.com/php/scm_man.php?file=lib/Utils.pm">Utils</a><br>
607 =head1 BUGS AND LIMITATIONS
609 There are no known bugs in this script
611 Please report problems to Andrew DeFaria <Andrew@ClearSCM.com>.
613 =head1 LICENSE AND COPYRIGHT
615 Copyright (c) 2010, ClearSCM, Inc. All rights reserved.