5 =head1 NAME $RCSfile: cleartasks.pl,v $
15 Andrew DeFaria <Andrew@ClearSCM.com>
23 Sun Jan 2 19:40:28 EST 2011
27 $Date: 2013/06/02 18:47:26 $
33 Usage cleartasks.pl: [-u|sage] [-ve|rbose] [-deb|ug]
36 -u|sage: Displays usage
39 -de|bug: Output debug messages
41 -da|emon: Run in daemon mode (Default: yes)
42 -p|idfile: File to be created with the pid written to it (Default:
43 cleartasks.pid). Note: pidfile is only written if -daemon is
48 Examine the Clearadm schedule and perform the tasks required.
58 use lib "$FindBin::Bin/lib", "$FindBin::Bin/../lib";
67 my $VERSION = '$Revision: 1.25 $';
68 ($VERSION) = ($VERSION =~ /\$Revision: (.*) /);
70 my $logfile = "$Clearadm::CLEAROPTS{CLEARADM_LOGDIR}/$FindBin::Script.log";
71 my $pidfile = "$Clearadm::CLEAROPTS{CLEARADM_RUNDIR}/$FindBin::Script.pid";
74 # Augment PATH with $Clearadm::CLEAROPTS{CLEARADM_BASE}
75 $ENV{PATH} .= ":$Clearadm::CLEAROPTS{CLEARADM_BASE}";
77 my ($clearadm, $clearexec);
79 sub HandleSystemNotCheckingIn (%) {
84 my $message = "Unable to connect to system $system{name}:$system{port}";
87 task => 'System checkin',
88 started => Today2SQLDatetime,
91 system => $system{name},
94 my ($err, $msg, $lastid) = $clearadm->AddRunlog (%runlog);
96 $clearadm->Error ("Unable to add to runlog (Status: $err)\n$msg") if $err;
98 # Check to see if we should notify anybody about this non-responding system
99 my %notification = $clearadm->GetNotification ('System checkin');
101 my $when = Today2SQLDatetime;
102 my $nomorethan = lc $notification{nomorethan};
103 my $systemLink = $Clearadm::CLEAROPTS{CLEARADM_WEBBASE};
104 $systemLink .= "/systemdetails.cgi?system=$system{name}";
105 my $runlogLink = $Clearadm::CLEAROPTS{CLEARADM_WEBBASE};
106 $runlogLink .= "/runlog.cgi?id=$lastid";
107 my $subject = "System is not responding (Is clearagent running?)";
110 <h1><font color="red">Alert</font> System not responding!</h1>
113 <p>On $when the system <a href="$systemLink">$system{name}</a> was <a
114 href="$runlogLink">not responding</a> to clearagent requests. This can happen if
115 clearagent is not setup and running on the system.</p>
128 verbose "$system{name}: $subject";
131 } # HandleSystemNotCheckingIn
133 sub SystemsCheckin () {
134 foreach ($clearadm->FindSystem) {
137 next if $system{active} eq 'false';
139 verbose "Contacting system $system{name}:$system{port}";
141 my $startTime = time;
143 my $status = $clearexec->connectToServer (
149 HandleSystemNotCheckingIn %system;
153 $clearexec->disconnectFromServer;
155 verbose 'Successfully checked in with system: '
156 . "$system{name}:$system{port}";
158 display __FILE__ . " DEBUG: System undefined 1" unless $system{name};
159 $clearadm->UpdateSystem (
161 (lastheardfrom => Today2SQLDatetime)
164 $clearadm->ClearNotifications ($system{name})
165 if $system{notification} and $system{notification} eq 'Heartbeat';
171 sub UpdateRunlog ($$$$) {
172 my ($status, $startTime, $task, $output) = @_;
175 task => $$task{name},
176 system => $$task{system},
177 started => Today2SQLDatetime,
180 $runlog{status} = $status;
184 $runlog{message} = join "\n", @$output;
186 $runlog{message} = 'Successful execution of ';
187 $runlog{message} .= "$$task{name}: $$task{command}";
191 $runlog{message} = join "\n", @$output;
193 $runlog{message} = 'Unable to execute ';
194 $runlog{message} .= "$$task{name}: $$task{command} ";
195 $runlog{message} .= join (' ', @$output);
199 my ($err, $msg, $lastid) = $clearadm->AddRunlog (%runlog);
201 $clearadm->Error ($msg, $err) if $err;
206 sub MakeSystemLink ($) {
209 return "$Clearadm::CLEAROPTS{CLEARADM_WEBBASE}/systemdetails.cgi?system="
213 sub MakeLoadavgLink ($) {
216 return "$Clearadm::CLEAROPTS{CLEARADM_WEBBASE}/plot.cgi?type=loadavg&system="
217 . "$system&scaling=Hour&points=24";
220 sub ProcessLoadavgErrors ($$$$@) {
221 # TODO: Also need to handle the case where the error was something other
222 # than "Load average over threshold". Perhaps by having different return
223 # status. Also, runlog entry #22169 never reported!
224 my ($notification, $task, $system, $lastid, @output) = @_;
226 my $when = Today2SQLDatetime;
229 # We need to log this output. Write it to STDOUT
232 my ($subject, $message, $currLoadavg, $threshold, $systemLink, $loadavgLink);
234 if (/System: (\w+) Loadavg (\d+\.\d+) Threshold (\d+\.\d+)/) {
238 $systemLink = MakeSystemLink $system;
239 $loadavgLink = MakeLoadavgLink $system;
240 $subject = "Load average of $currLoadavg exceeds threshold ";
241 $subject .= "($threshold)";
244 <h1><font color="red">Alert</font> Load Average is over the threshold!</h1>
247 <p>On $when the system <a href="$systemLink">$system</a>'s load avg
248 (<a href="$loadavgLink">$currLoadavg</a>) had exceeded the threshold set for
249 this system ($threshold).</p>
251 } elsif (/ERROR.*system\s+(\S+):/) {
253 $systemLink = MakeSystemLink $system;
254 $subject = "Error trying to obtain Loadavg";
257 <h1><font color="red">Alert</font> Unable to obtain Loadavg!</h1>
260 <p>On $when we were unable to obtain the Loadavg for
261 system <a href="$systemLink">$system</a>.</p>
263 <p>The following was the error message:</p>
268 <p>On $when on the system $system, we were unable to parse the Loadavg output. This is what we saw:</p>
272 $message .= join "\n", @output;
273 $message .= "</pre>";
274 $clearadm->Error ($message, -1);
291 } # ProcessLoadAvgErrors
293 sub ProcessFilesystemErrors ($$$$@) {
294 # TODO: Also need to handle the case where the error was something other
295 # than "Filesystem over threshold". Perhaps by having different return
297 my ($notification, $task, $system, $lastid, @output) = @_;
299 my $when = Today2SQLDatetime;
304 # We need to log this output. Write it to STDOUT
307 if (/System:\s*(\S+)\s*Filesystem:\s*(\S+)\s*Used:\s*(\d+\.\d+)%\s*Threshold:\s*(\d+)/) {
315 $system{$1} = [$system{$1}, \%fsinfo];
317 $system{$1} = \%fsinfo;
322 foreach my $systemName (keys %system) {
325 if (ref $system{$systemName} eq 'HASH') {
326 push @fsinfo, $system{$systemName};
328 push @fsinfo, @{$system{$systemName}};
331 my $systemLink = MakeSystemLink ($systemName);
332 my $subject = 'Filesystem has exceeded threshold';
333 my $message = <<"END";
335 <h1><font color="red">Alert</font> Filesystem is over the threshold!</h1>
338 <p>On $when the following filesystems on <a href="$systemLink">$systemName</a>
339 were over their threshold.</p>
345 my $filesystemLink = $Clearadm::CLEAROPTS{CLEARADM_WEBBASE};
346 $filesystemLink .= "/plot.cgi?type=filesystem&system=$systemName";
347 $filesystemLink .= "&filesystem=$fsinfo{filesystem}";
348 $filesystemLink .= '&scaling=Day&points=7';
349 $message .= "<li>Filesystem <a href=\"$filesystemLink\">";
350 $message .= "$fsinfo{filesystem}</a> is $fsinfo{usedPct}% full. Threshold is ";
351 $message .= "$fsinfo{threshold}%</li>";
368 } # ProcessFilesystemErrors
370 sub NonZeroReturn ($$$$$$) {
371 my ($system, $notification, $status, $lastid, $output, $task) = @_;
373 my @output = @{$output};
376 my $when = Today2SQLDatetime;
378 my $subject = "Non zero return from $task{command} "
379 . "executing on $system";
380 my $taskLink = $Clearadm::CLEAROPTS{CLEARADM_WEBBASE};
381 $taskLink .= "/tasks.cgi?task=$task{name}";
382 my $similarLink = $Clearadm::CLEAROPTS{CLEARADM_WEBBASE};
383 $similarLink .= "/runlog.cgi?system=$task{system}"
385 . "&task=$task{name}";
386 my $runlogLink = $Clearadm::CLEAROPTS{CLEARADM_WEBBASE};
387 $runlogLink .= "/runlog.cgi?id=$lastid";
388 my $message = <<"END";
390 <h1><font color="red">Alert</font> Non zero status from script execution!</h1>
393 <p>On $when, while executing <a href="$taskLink">$task{name}</a> on
394 $task{system}, a non zero status of $status was returned. Here is the resulting
395 output:</p><blockquote><pre>
398 $message .= join "\n", @output;
401 <p>You may wish to examine the individual <a href="$runlogLink">runlog entry</a>
402 that caused this alert or a list of <a href="$similarLink">similar
406 $message .= "</pre></blockquote>";
421 sub ExecuteTask ($%) {
422 my ($sleep, %task) = @_;
424 my ($status, @output, %system, $subject, $message);
426 verbose_nolf "Performing task $task{name}";
428 my %notification = $clearadm->GetNotification ($task{notification});
430 my $startTime = time;
432 if ($task{system} =~ /localhost/i) {
433 verbose " on localhost";
434 ($status, @output) = Execute "$task{command} 2>&1";
436 %system = $clearadm->GetSystem ($task{system});
438 verbose " on $system{name}";
440 $status = $clearexec->connectToServer (
446 $output[0] = "Unable to connect to system $system{name}:$system{port} to "
447 . "execute $task{command}";
450 ($status, @output) = $clearexec->execute ($task{command});
452 $output[0] = "Unable to exec $task{command} on $system{name}"
456 $clearexec->disconnectFromServer;
459 my $lastid = UpdateRunlog ($status, $startTime, \%task, \@output);
462 if ($notification{cond}
463 and $notification{cond} =~ /non zero return/i) {
472 } elsif ($notification{cond} =~ /loadavg over threshold/i) {
473 ProcessLoadavgErrors ($notification{name}, $task{name}, $system{name}, $lastid, @output);
474 } elsif ($notification{cond} =~ /filesystem over threshold/i) {
475 ProcessFilesystemErrors ($notification{name}, $task{name}, $system{name}, $lastid, @output);
478 $clearadm->ClearNotifications ($task{system});
481 my ($err, $msg) = $clearadm->UpdateSchedule (
483 ( 'lastrunid' => $lastid ),
486 $clearadm->Error ($msg, $err) if $err;
488 $sleep -= time - $startTime;
495 'usage' => sub { Usage },
496 'verbose' => sub { set_verbose },
497 'debug' => sub { set_debug },
498 'daemon!' => \$daemon,
499 'pidfile=s' => \$pidfile,
500 ) or Usage "Invalid parameter";
502 Usage 'Extraneous options: ' . join ' ', @ARGV
505 EnterDaemonMode $logfile, $logfile, $pidfile
508 display "$FindBin::Script V$VERSION started at " . localtime;
510 $clearadm = Clearadm->new;
511 $clearexec = Clearexec->new;
513 $clearadm->SetNotify;
516 # First check in with all systems
519 my ($sleep, @workItems) = $clearadm->GetWork;
521 foreach (@workItems) {
522 my %scheduledTask = %{$_};
524 $scheduledTask{system} ||= 'All systems';
526 if ($scheduledTask{system} =~ /all systems/i) {
527 foreach my $system ($clearadm->FindSystem) {
528 $scheduledTask{system} = $$system{name};
529 $sleep = ExecuteTask $sleep, %scheduledTask;
532 $sleep = ExecuteTask $sleep, %scheduledTask;
537 verbose "Sleeping for $sleep seconds";
544 =head1 CONFIGURATION AND ENVIRONMENT
546 DEBUG: If set then $debug is set to this level.
548 VERBOSE: If set then $verbose is set to this level.
550 TRACE: If set then $trace is set to this level.
558 L<Getopt::Long|Getopt::Long>
560 =head2 ClearSCM Perl Modules
576 <a href="http://clearscm.com/php/scm_man.php?file=clearadm/lib/Clearadm.pm">Clearadm</a><br>
577 <a href="http://clearscm.com/php/scm_man.php?file=clearadm/lib/Clearexec.pm">Clearexec</a><br>
578 <a href="http://clearscm.com/php/scm_man.php?file=lib/DateUtils.pm">DateUtils</a><br>
579 <a href="http://clearscm.com/php/scm_man.php?file=lib/Display.pm">Display</a><br>
580 <a href="http://clearscm.com/php/scm_man.php?file=lib/TimeUtils.pm">TimeUtils</a><br>
581 <a href="http://clearscm.com/php/scm_man.php?file=lib/Utils.pm">Utils</a><br>
586 =head1 BUGS AND LIMITATIONS
588 There are no known bugs in this script
590 Please report problems to Andrew DeFaria <Andrew@ClearSCM.com>.
592 =head1 LICENSE AND COPYRIGHT
594 Copyright (c) 2010, ClearSCM, Inc. All rights reserved.