#!/usr/bin/perl ####################################################################### # # # SysReport. # # # # This version is derived from the original program by Paul Grosse. # # This version Copyright (c)2007 Future publishing. # # # # Program that polls a number of resources every hour on a crontab # # and each day, at 6 am, emails a report. In addidion, if one of # # those resources is over a limit, it emails an emergency notice # # anyway. # # # # There are plenty of remarks on what each part of the program # # does, therefore, you can work out how to modify it for # # yourself using the examples here. # # # ####################################################################### use warnings; # Remember to make sure that this file is executable (cd to its directory # and use 'chmod 700 sysreport' ). # Also, you need to make sure that you have sendmail working. # Program outline... # # make note of the current time # # check each resource against limits # # send mail if resource over limit # # log resouce values # # if report time of day, send email with report in it for last 24 hours # we want to know: # time; (the timeof day and the date so that the log makes sense) # current PID; (this can be used to see when there are a lot of processes being created) # No of processes; (this can be used to make sure that superservers haven't gone mad) # No of mailtrpts; (this is just an example super server child) # length of smtp log; (this is it's xinetd log) # length of httpdeflt log; (this is a virtual web server log) # eth0 RX; (this is network interface's received bytes) # eth0 TX (this is network interface's transmitted bytes) # get the time - if this is the first thing we do, we don't have a race condition for it. # use localtime and select a substring of it $dttm = substr(localtime, 8, 2)."/".substr(localtime, 4, 3)."/".substr(localtime, 22, )." ".substr(localtime, 11, 5); # create another copy of the hour - this is just lazy $ctime = substr(localtime, 11, 2); # this is the current hour. Check it later on to see # if the report needs outputting ## current PID; $$ is Perl's PID for this program $cpid = $$; #Note that there are no limits with this - it is just a number that cycles ## No processes; Get this by using 'ps ax' and piping it to 'wc -l' so that we just get a number $cl = "ps ax | wc -l"; # execute the string and store the output in a scalar (it is just one number on one line) $cnp = (readpipe ($cl))[0]; # extract the number using a regular expression - the brackets put the recognised digit(s) # into a special variable $cnp =~ /(\d+)/; # reuse the scalar so that now, it contains (guaranteed to contain) just the number $cnp = $1; #limit - set the limit for the number of processes we think is acceptable for this machine $cnpl = 250; # Now, repeat the process with other properties of the system # You can copy the $cl (command line) to a console and execute # it to see what the output looks like with any of these ## No mailtrpts; $cl = "pgrep mailtrpt | wc -l"; $cntp = (readpipe ($cl))[0]; $cntp =~ /(\d+)/; $cntp = $1; #limit $cntpl = 45; ## length of smtp log; #With this one, the output is several lines so we use 'grep' to # sort out those lines with the right file names. as there are a number of them, # we store them in an array. $cl = "ls -al /var/log/mailtar | grep mailtrpt"; @ls = readpipe ($cl); # however, we do know that the one we want is the first in the list. $ls[0] =~ m/root\s+(\d+)\s/; $mailtar = $1; #limit $mailtarl = 20000000; ## length of httpdeflt log; $cl = "ls -al /var/log/httpd | grep httpdeflt_log"; @ls = readpipe ($cl); $ls[0] =~ m/root\s+(\d+)\s/; $pgac = $1; #limit $pgacl = 10*1024*1024; ##eth0 rx and tx bytes $cl = "ifconfig eth0"; @ls = readpipe ($cl); # with this one, both of the results are on the same line so we can take them # using just one regular expression - the two portions in brackets are the results $ls[7] =~ m/RX bytes:(\d+)\s.+TX bytes:(\d+)\s/; # and the first one is stored in $1 and the second in $2 $et0r = $1; $et0t = $2; #Note that there are no limits with this - it is just a number #Now that we have all of the current results that we need, we can check for excesses # by comparing each with the variables of the same name but that have an additional # letter 'l' at the end - signifying the limit # Here, if the value is greater than the limit, we generate two strings: $warng (the short warning) # and $warngm (the message to appear in the email. Note that the second string goes over a number of lines. if ($cnp > $cnpl) { $warng = "Total Processes ($cnp) Exceeds limit"; $warngm = "$dttm Total number of processes exceeds the limit of $cnpl. Current number of processes = $cnp."; # At the end, we call the process that sends the mails. &send_warning; }; if ($cntp > $cntpl) { $warng = "Total mailtrpts ($cntp) Exceeds limit"; $warngm = "$dttm Total number of tarpits exceeds the limit of $cntpl. Current number of tarpits = $cntp."; &send_warning; }; # Here, we format the values because we divide them by 1024 (we # are talking about disc space here so we use Ki instead of the SI k) if ($mailtar > $mailtarl) { $warng = "mailtrpt log ($mailtar) Exceeds limit"; $fmailtar = int($mailtar / 1024); $fmailtarl = int($mailtarl / 1024); $warngm = "$dttm Length of tarpit log exceeds the limit of $fmailtarl"."KiB. Current length of tarpit log = $fmailtar"."KiB."; &send_warning; }; if ($pgac > $pgacl) { $warng = "httpdeflt log ($pgac) Exceeds limit"; $fpgac = int($pgac / 1024); $fpgacl = int($pgacl / 1024); $warngm = "$dttm Length of httpdeflt log exceeds the limit of $fpgacl"."KiB. Current length of httpdeflt log = $fpgac"."KiB."; &send_warning; }; # Now that we have sent all of the values that we need to, we can log them. # Note that the log file path is absolute so you will have to edit this to # whereever you need it to be. The reason for it being absolute is that the # program will be called from the crontab. Also, note that the log file is # appended by using the double right arrow '>>'. # log the values open(SRL, ">>/root/bin/perl/sysrep/sysreport/sysrep_log"); print SRL "$dttm $ctime $cpid $cnp $cntp $mailtar $pgac $et0r $et0t\n"; close(SRL); # Now that we have sent any warnings by email, we can see if we need to write a report. #if time set to 6am, send daily summary. note that you can set this to any time you like. # Alternatively, you can make it do it twice a day by checking for two times like so # if (($ctime eq '08') or ($ctime eq '15')) { # To use the above line, remove the hash mark '#' but also make sure that you remark # out the line below by starting it with a hash mark - this way will make sure that # there is only one opening brace '{' if ($ctime eq '19') { #send daily summary #open log file open(SRL, "; close(SRL); #keep last 25 entries foreach $x(0..25) { $lines[$x] = $lg[$x-26]; } #clean up entries foreach (@lines) { chomp; }; # Now to formulate the report - we'll build a string containing the message so that # we only need to pipe it to the sendmail program. $msg = "From: SysReport To: Systems Administrator Subject: ** Daily Report: $dttm - previous 24 hours ** +-------------------------------------+ | SYSTEM REPORT for $dttm | +-------------------------------------+ CURRENT STATUS:-\n"; #find uptime and include that as well... $cl = "uptime"; $utp = (readpipe ($cl))[0]; $utp =~ m/up (\d+) days\s+(\d+):(\d\d),/; ($utpd, $utph, $utpm) = ($1, $2, $3); # make sure that we have a singular if the machine has only been up for one day if ($utpd == 1) {$utpdd = "day"} else {$utpdd = "days"}; $msg .= ' ' x 23 ."System up for: $utpd $utpdd, ".$utph."h ".$utpm."m Current -Number of- -Log Size KiBytes-- ----eth0 kBytes---- Date Time PID Procs Trpts mailtrpt httpdeflt RX TX\n"; # That's the headers printed out for the current status line. $fcpid = substr(' '.$cpid, -5); $fcnp = substr(' '.$cnp, -4); $fcntp = substr(' '.$cntp, -3); # Now to use commas to make it easy to read long numbers. To do this, we use the # comma-ise function - see at the bottom # Again, disc space uses 1024 ... $fmailtar = substr(' ' x 7 .commaise(int($mailtar/1024)), -7); $fpgac = substr(' ' x 7 .commaise(int($pgac/1024)), -7); # ... and non-disc space quantities use SI (k = 1000) $ket0r = int($et0r/1000); $fket0r = substr(' ' x 9 .commaise($ket0r), -9); $ket0t = int($et0t/1000); $fket0t = substr(' ' x 9 .commaise($ket0t), -9); # Now that we have these values, we'll add them to our string. $msg .= "$dttm $fcpid $fcnp $fcntp $fmailtar $fpgac $fket0r $fket0t SUMMARY of Last 24 hours:- Delta Number of Log Deltas/K eth0 deltas/k Hourly PID procs tarpits mailtrpt httpdeflt RX TX\n"; # These are going to print out the Max, avg, min and, where applicable, total # values for each of these. # There are several ways of doing this and both of them are messy. # You can either: # go through each record, making notes of the max, min and total for # each one you are interested in one at a time, then build up the string # line by line; or, # you can create a variable for each line and then go through each heading, # simultaneously collecting data for each column's max, min and total. # Also, you could create a 3D array and do it that way. However, we'll do it # column by column for each line. # # Here are the base strings for each value we are intersted in... $mx = " Max "; $av = " Avg "; $mn = " Min "; $tot = "Totals "; # We are only intersted in columns 3, 4, 5, 6, 7, 8 and 9 so we'll use the quote-white # to create a list to use. foreach $x (qw/ 3 4 5 6 7 8 9 /) { #Set total to zero $totv = 0; #Next, we have the number of values in the total (this is used because if we have a # log that is archived, that would create a negative number which would upset # everything else. $valsv = 0; # create an old version of the line's array. @lgl1 = split " ", $lines[1]; #Now, we have to go through 24 lines and in some cases, compare them with the previous foreach $y (2..$#lines) { #copy the old one - this is why we prepared it before we started the foreach loop @lgl0 = @lgl1; @lgl1 = split " ", $lines[$y]; unless (($x == 4) or ($x == 5)) { # not pid or tarpits (ie, a delta) # compare with previous result in the log $v = $lgl1[$x] - $lgl0[$x]; if ($v < 0) { #rollover if ($x == 3) { #pid -- needs a rollover adding to it $v += 0x8000; } elsif (($x == 8) or ($x == 9)) { $v += 0x80000000; $v += 0x80000000; } #if it is still negative, it is a log that is reset so we'll look out for that } } else { #not deltas $v = $lgl1[$x]; } #we have $v so now, let us put it into its max/min et cetera if ($y == 2) { #we are at the beginning so initialise max and min $mxv = $mnv = $v; } # if the max value is less than the current value, update it if ($v > $mxv) {$mxv = $v}; # if the current value is greater than zero [ie, not negative] and (less # than the minimum [being positive, it qualifies as being the new minimum] # or the minimum is less than zero [it hasn't been update by a positive # number yet] ), then make it the new minimum. if ($v > 0) { if (($mnv < 0) or ($mnv > $v)) { $mnv = $v; } } # if the number is greater than zero then if ($v >= 0) { #add it to the total $totv += $v; # and add one to the number of values we have added to the total value $valsv++; } # Now, loop back to do the next line for this record } #we've got a total, max and min so, let's calculate the average using # the total and the number of values in it. $avv = int($totv / $valsv); # Now, we need to format the output for each value if ($x == 3) { #delta pid $mxv = substr(' ' x 7 .commaise($mxv), -7); $mnv = substr(' ' x 7 .commaise($mnv), -7); $avv = substr(' ' x 7 .commaise($avv), -7); $totv = substr(' ' x 7 .commaise($totv), -7); } elsif (($x == 4) or ($x == 5)) { #procs and tarpits $mxv = substr(' ' x 7 .commaise($mxv), -8); $mnv = substr(' ' x 7 .commaise($mnv), -8); $avv = substr(' ' x 7 .commaise($avv), -8); $totv = ' ' x 7 . '-'; } elsif (($x == 6) or ($x == 7)) { #mailtar and httpdeflt logs $mxv = int($mxv/1024); $mxv = substr(' ' x 10 .commaise($mxv), -10); $mnv = int($mnv/1024); $mnv = substr(' ' x 10 .commaise($mnv), -10); $avv = int($avv/1024); $avv = substr(' ' x 10 .commaise($avv), -10); $totv = int($totv/1024); $totv = substr(' ' x 10 .commaise($totv), -10); } else { #eth0 RX and TX $mxv = int($mxv/1000); $mxv = substr(' ' x 9 .commaise($mxv), -9); $mnv = int($mnv/1000); $mnv = substr(' ' x 9 .commaise($mnv), -9); $avv = int($avv/1000); $avv = substr(' ' x 9 .commaise($avv), -9); $totv = int($totv/1000); $totv = substr(' ' x 9 .commaise($totv), -9); } # Now, concatenate those strings with the four substrings we are accumulating $mx .= $mxv; $av .= $avv; $mn .= $mnv; $tot .= $totv; # Now, cycle back to the next field } # Now, we have all of the values we want, formatted the way we want so add them # to our string. $msg .= "$mx $av $mn Day's $tot LOG for Last 25 hours:- Tot Delta mailtrpt log Tar httpdeflt log eth0 deltas/k Date Time Prc PID size/K Delta pit Size/K Delta RX TX\n"; # Now, we have the headers for the previous 25 values. This is a bit easier. We still do it for # each line of the log but we only have to do it once. #Prepare the previous log line as before... @lgl1 = split " ", $lines[0]; #Now, start cycling through them foreach $x (1..$#lines) { #copy the ld line and split the new one @lgl0 = @lgl1; @lgl1 = split " ", $lines[$x]; #add the date to the line #first, check its length if (length ($lgl1[0]) == 8) {$lgl1[0] = " " . $lgl1[0]}; $msg .= "$lgl1[0] $lgl1[1] "; #add the procs $msg .= substr(' ' x 3 . $lgl1[4], -3); #add the delta pid # Note that we can get around it being negative by adding 0x8000 to it and then modding by the same $delpid = substr(' ' x 6 .commaise(($lgl1[3] - $lgl0[3] + 0x8000) % 0x8000), -6); $msg .= "$delpid "; #add the mailtar size $fmailtar = substr(' ' x 7 . commaise(int($lgl1[6]/1024)), -7); $msg .= "$fmailtar "; #add the mailtar delta, noting that if we have a negative number, it is a reset if ($lgl1[6] < $lgl0[6]) { #log file has been reset $dfmailtar = " reset "; } else { # use 1024 because this is disc related. However, we want it to go to a 1/10th of a KiB $dfmailtar = substr(' ' x 7 .commaise(int(($lgl1[6]-$lgl0[6])/102.4)/10), -7); # also, note that if the end is whole (ie, no '.0') then add it to it and strip the front off the string. unless ($dfmailtar =~ m/\./) {$dfmailtar = substr($dfmailtar . ".0",-7)} }; $msg .= "$dfmailtar "; #add the tarpits $msg .= substr(' ' x 3 . $lgl1[5], -3); #add the httpdeflt size $fpal = substr(' ' x 7 .commaise(int($lgl1[7]/1024)), -7); $msg .= "$fpal "; #add the pal delta if ($lgl1[7] < $lgl0[7]) { #log file has been reset $dfpal = " reset "; } else { $dfpal = substr(' ' x 7 .commaise(int(($lgl1[7]-$lgl0[7])/102.4)/10), -7); unless ($dfpal =~ m/\./) {$dfpal = substr($dfpal . ".0",-7)} }; $msg .= "$dfpal "; # Now, add the eth0 RX delta if ($lgl1[8] > $lgl0[8]) { $drx = substr(' ' x 6 .commaise(int(($lgl1[8] - $lgl0[8])/ 1000)), -6); } else { $drx = $lgl1[8] - $lgl0[8]; # we can't add 0x100000000 in one go because it is too large on some machines so # let's add half of it twice $drx += 0x80000000; $drx += 0x80000000; $drx = substr(' ' x 6 .commaise(int(($drx)/ 1000)), -6); } $msg .= "$drx "; #add the eth0 TX delta if ($lgl1[9] > $lgl0[9]) { $dtx = substr(' ' x 7 .commaise(int(($lgl1[9] - $lgl0[9])/ 1000)), -7); } else { $dtx = $lgl1[9] - $lgl0[9]; $dtx += 0x80000000; $dtx += 0x80000000; $dtx = substr(' ' x 7 .commaise(int(($dtx)/ 1000)), -7); } $msg .= "$dtx"; # That's the end of that line so let's add a next line character. $msg .= "\n"; # Cycle around for the next. } # That is a long table for an email so let's add a fotter to it to reiterate the columns $msg .= " Date Time Tot Delta mailtrpt log Tar httpdeflt log Eth0 deltas/k Prc PID size/K Delta pit Size/K Delta RX TX\n"; # These are debug lines that you can use to debug the program when you edit it for your own system. # Just unremark the print line and remark the file lines. If you want to stop the program there, # unremark the exit as well (the 2 is the exit code - you can use these to see how your program ends) # Remember to unremark the file commands and re-remark the print and exit lines. #print $msg;######################################### DEBUG ################################ #exit (2); ######################################### DEBUG ################################ open (SM, "|/usr/lib/sendmail -t -i"); print SM $msg; close (SM) or warn "Sendmail did not close nicely"; } #And, that is it. sub commaise { # This sub takes the contents of $_ and adds commas every third number from the right. my $a = shift; 1 while ($a =~ s/(\d+)(\d\d\d)/$1,$2/); return $a; } sub send_warning { # This subroutine creates a message and replaces two substrings with the warning and # warning message that are created just before it is called. $msg = "From: SysReport To: Systems Administrator Subject: ** WARNING: ASASA WARNING ASASB \n"; $msg =~ s/ASASA/$warng/; $msg =~ s/ASASB/$warngm/; # debug lines as before. #print $msg;######################################### DEBUG ################################ #exit (2); ######################################### DEBUG ################################ open (SM, "|/usr/lib/sendmail -t -i"); print SM $msg; close (SM) or warn "Sendmail did not close nicely"; }