[prev in list] [next in list] [prev in thread] [next in thread] 

List:       sidewinder
Subject:    Re: [Sidewinder] script for moitoring processes on the Sidewinder
From:       sidewinder-admin () adeptech ! com
Date:       2003-12-16 6:59:00
[Download RAW message or body]

i've attached a perl script that will check the cpu utilization of processes (using \
"ps auxww") every 60 seconds, calculate an average, and generate a report every 5 \
checks (5 minutes).  this particular script does not currently "daemonize" itself so \
if you chose to use it, you'll probably want to run it with the proper output \
redirection.  here is a bourne shell example:

./ps-check.pl  > /var/log/ps-check.log 2>&1 &

some other notes/comments:

- it is *not* meant to be run out of cron; it is supposed to run continuously by \
                itself.
- be sure to change the e-mail addressez and modify the cpu utilization threshold in \
                the global_hash section near the top of the script.
- i cut this out of a more elaborate distributed monitoring script, so forgive the \
leftover complexity and lack of comments within the code.

i hope this helps someone!

randy blahut
rblahut@aol.com

In a message dated 12/11/2003 9:50:28 AM Eastern Standard Time, \
sidewinder-admin@adeptech.com writes:

> I have been trying to create a perl script for monitoring processes on the \
> Sidewinder.  I wanted to have it send an e-mail for any process that exceed X% CPU \
> and I was going to run it from crontab every 5 minutes.  My perl skills are sadly \
> lacking so I was wondering if anyone has already written a script that does that or \
> something similar.   If it is a shell script or awk script that would be fine too.  \
> I can stumble my way through modifying those without hurting my brain too badly =) \
> Also, does anyone know of a good site on the Internet  for perl or awk systems \
> administration scripts? 
> Regards,
> Jeffery Gieser


["ps-check.pl" (application/octet-stream)]

#!/usr/bin/perl

$stdin = *STDIN;
$stdout = *STDOUT;
$stderr = *STDERR;

$eol = "\n";
$tab = "\t";
$ERROR_STRING = '';
$VERSION = '1.0.0';

%global_hash = (
	'ps_cmd'	=>	'ps auxww',
	'mail_cmd'	=>	'mail',
	'mail_subj'	=>	'process cpu utilization exceeded threshold',
	'mail_to'	=>	[
		'root',
		'admin@somewhere.com',	# add e-mail addressez here as necessary
	],
	'thresh'	=>	80,	# cpu utilization threshold
	'check_t'	=>	60,	# interval of secondz bewteen checkz
	'max_hist'	=>	5,	# intervalz before report
);
$global_hash{'report_t'} = $global_hash{'max_hist'};

# signalz

$SIG{'INT'} = \&handle_signals;
$SIG{'CHILD'} = \&handle_signals;
$SIG{'TERM'} = \&handle_signals;
$SIG{'PIPE'} = \&handle_signals;
#$SIG{'PIPE'} = IGNORE;

# main

do {
	&error_exit;
} unless (&check_loop (\%global_hash));

&clean_exit;

# subroutinez

sub check_loop {
	my ($config_hashref) = shift (@_);
	my ($run_flag) = 1;
	my ($check_count) = 0;
	my (%data_hash) = ();
	my (
		$pid,
	);
	while ($run_flag) {
		$check_count ++;
		do {
			return (0);
		} unless (
			&check_ps (
				$config_hashref->{'ps_cmd'},
				\%data_hash,
				$config_hashref->{'thresh'},
				$config_hashref->{'max_hist'},
			)
		);
		if ($check_count == $config_hashref->{'report_t'}) {
			# report on processez that exceeded thresholdz
			do {
				return (0);
			} unless (
				&mail_report (
					$config_hashref,
					\%data_hash,
				)
			);
			$check_count = 0;
		}
		sleep ($config_hashref->{'check_t'});
	}
	return (1);
}

sub mail_report {
	my ($config_hashref) = shift (@_);
	my ($hashref) = shift (@_);
	my (@pids) = sort { $a <=> $b } (keys (%{$hashref}));
	my ($mail_pipe) = join (
		'',
		'| ',
		$config_hashref->{'mail_cmd'},
		' -s \'',
		$config_hashref->{'mail_subj'},
		': ',
		scalar (localtime (time)),
		'\' ',
		join (' ', @{$config_hashref->{'mail_to'}}),
	);
	my (
		$pid,
	);
	# return immediately if there is nothing to report
	do {
		return (1);
	} unless (scalar (@pids));
	do {
		$ERROR_STRING = join ('', 'could not execute ', $config_hashref->{'mail_cmd'}, ': ', $!);
		return (0);
	} unless (open (MAIL, $mail_pipe));
	print MAIL join (
		$tab,
		'pid',
		'name',
		'cpu-average',
	), $eol;
	foreach $pid (@pids) {
		print MAIL join (
			$tab,
			$pid,
			$hashref->{$pid}{'name'},
			$hashref->{$pid}{'avg_cpu'},
		), $eol;
	}
	close (MAIL);
	return (1);
}

sub check_ps {
	my ($cmd) = shift (@_);
	my ($hashref) = shift (@_);
	my ($thresh) = shift (@_);
	my ($max_hist) = shift (@_);
	my (%ps_hash) = ();
	my (
		$pid,
	);
	do {
		return (0);
	} unless (&parse_ps ($cmd, \%ps_hash));
	foreach $pid (keys (%ps_hash)) {
		# skip kernel idle thread (not applicable on sidewinder 5.2.1.x)
		next if ($ps_hash{$pid}{'name'} =~ /^\(?idle\)?$/o);
		if ($ps_hash{$pid}{'cpu'} >= $thresh) {
			# create new hash structure as necessary
			do {
				$hashref->{$pid} = {
					'name'		=>	$ps_hash{$pid}{'name'},
					'avg_cpu'	=>	0,
					'hist'		=>	[],
				};
			} unless (defined ($hashref->{$pid}));
			# add history
			push (@{$hashref->{$pid}{'hist'}}, $ps_hash{$pid});
			# remove old history
			if ((scalar (@{$hashref->{$pid}{'hist'}})) > $max_hist) {
				shift (@{$hashref->{$pid}{'hist'}});
			}
			# calculate cpu average
			&calc_avg (
				$hashref->{$pid}{'hist'},
				'cpu',
				\$hashref->{$pid}{'avg_cpu'},
			);
		} else {
			# remove pid from history if average is below threshold
			if (defined ($hashref->{$pid})) {
				# calculate cpu average
				&calc_avg (
					$hashref->{$pid}{'hist'},
					'cpu',
					\$hashref->{$pid}{'avg_cpu'},
				);
				if ($hashref->{$pid}{'avg_cpu'} < $thresh) {
					delete ($hashref->{$pid});
				}
			}
		}
	}
	return (1);
}

sub calc_avg {
	my ($arrayref) = shift (@_);
	my ($key) = shift (@_);
	my ($avg_ref) = shift (@_);
	my ($total) = 0;
	my (
		$hashref,
	);
	foreach $hashref (@{$arrayref}) {
		$total += $hashref->{$key};
	}
	${$avg_ref} = $total / scalar (@{$arrayref});
	return (1);
}

sub parse_ps {
	my ($cmd) = shift (@_);
	my ($hashref) = shift (@_);
	my (@parts) = ();
	my (
		$output,
		$base,
	);
	do {
		$ERROR_STRING = join ('', 'could not execute ', $cmd, ': ', $!);
		return (0);
	} unless ($output = `$cmd`);
	foreach $line (split ($eol, $output)) {
		next if ($line =~ /^\s*USER/o);
		(@parts) = split (/\s+/, $line);
		next unless ((scalar (@parts)) >= 9);
		$base = $parts[10];
		# attempt to strip certain charz from process name
		$base =~ s/[\/\(\[]?([\w\-\.]+)[:\)\]]?$/$1/o;
		%{$hashref->{$parts[1]}} = (
			'cpu'	=>	$parts[2],
			'mem'	=>	$parts[3],
			'rss'	=>	$parts[5],
			'time'	=>	$parts[9],
			'name'	=>	$base,
			'cmd'	=>	join (' ', @parts[10 .. $#parts]),
		);
	}
	return (1);
}

# support routinez

sub log_message {
	print $stderr join ('', @_), $eol;
	return (1);
}

sub clean_exit {
	my ($status) = shift (@_) || 0;
	exit ($status);
}

sub error_exit {
	my ($status) = shift (@_) || 1;
	&log_message ($ERROR_STRING);
	&clean_exit ($status);
}

sub handle_signals {
	my ($signal) = shift (@_);
	print $stderr 'received signal ', $signal, $eol;
	&clean_exit;
}

# end

_______________________________________________
Sidewinder mailing list
Sidewinder@adeptech.com
http://mail.adeptech.com/mailman/listinfo/sidewinder

[prev in list] [next in list] [prev in thread] [next in thread]