#!/usr/local/bin/perl
# monitor.pl
# Check all the monitors and send email if something is down

$no_acl_check++;
delete($ENV{'FOREIGN_MODULE_NAME'});
delete($ENV{'SCRIPT_NAME'});
delete($ENV{'SERVER_ROOT'});
require './status-lib.pl';

# Check if the monitor should be run now
@tm = localtime(time());
@hours = split(/\s+/, $config{'sched_hours'});
!@hours || &indexof($tm[2], @hours) >= 0 || exit;
@days = split(/\s+/, $config{'sched_days'});
!@days || &indexof($tm[6], @days) >= 0 || exit;

# Open status and number of fails files
&lock_file($oldstatus_file);
&read_file($oldstatus_file, \%oldstatus);
&lock_file($fails_file);
&read_file($fails_file, \%fails);

# Check for services that are down
$now = localtime(time());
$thishost = &get_system_hostname();
@services = &list_services();
$ecount = 0;
foreach $serv (@services) {
	if ($serv->{'nosched'} == 1) {
		# Scheduled checking totally disabled
		delete($oldstatus{$serv->{'id'}});
		next;
		}

	# Find the current status
	$warn = $serv->{'nosched'} == 0 ? $config{'sched_warn'} :
		$serv->{'nosched'} - 2;
	$stat = &service_status($serv);
	$o = $oldstatus{$serv->{'id'}};

	# Find the notification modes
	%notify = map { $_, 1 } split(/\s+/, $serv->{'notify'});

	# If the number of fails before warning is > 1, then the status may
	# still be considered OK even if it is down right now
	local $up = $stat->{'up'};
	if ($up != 1 && $serv->{'fails'} > 1) {
		$fails{$serv->{'id'}}++;
		if ($fails{$serv->{'id'}} < $serv->{'fails'}) {
			# Not really down yet
			$up = 1;
			}
		}
	else {
		$fails{$serv->{'id'}} = 0;
		}

	# Work out the hostname
	local $host = $serv->{'remote'} || $thishost;

	$thisemail = undef;
	if ($warn == 0 && $up == 0 && $o) {
		# Service has just gone down
		$subj = "$serv->{'desc'} down on $host";
		if ($notify{'pager'}) {
			$pager_msg .= "$host: \"$serv->{'desc'}\" is down $now! ";
			}
		if ($notify{'snmp'}) {
			push(@snmp_msg, "$host: $serv->{'desc'}");
			}
		$out = &run_on_command($serv, $serv->{'ondown'});
		if ($notify{'email'}) {
			$thisemail .= "Monitor on $host for '$serv->{'desc'}'\n".
			 "has detected that the service has gone down at $now\n";
			if ($out) {
				$thisemail .= $out;
				}
			$thisemail .= "\n";
			$ecount++;
			}
		}
	elsif ($warn == 1 && $up != $o &&
	       (defined($o) || $up == 0)) {
		# Service has changed status
		if ($up == 0) {
			$subj = "$serv->{'desc'} down on $host";
			if ($notify{'pager'}) {
				$pager_msg .= "$host: \"$serv->{'desc'}\" is down $now! ";
				}
			if ($notify{'snmp'}) {
				push(@snmp_msg, "$host: $serv->{'desc'}");
				}
			$out = &run_on_command($serv, $serv->{'ondown'});
			if ($notify{'email'}) {
				$thisemail .= "Monitor on $host for '$serv->{'desc'}'\n".
				 "has detected that the service has gone down at $now\n";
				if ($out) {
					$thisemail .= $out;
					}
				$thisemail .= "\n";
				$ecount++;
				}
			}
		elsif ($up == 1) {
			$subj = "$serv->{'desc'} back up on $host";
			if ($notify{'pager'}) {
				$pager_msg .= "$host: \"$serv->{'desc'}\" is back up $now! ";
				}
			if ($notify{'snmp'}) {
				push(@snmp_msg, "$host: $serv->{'desc'} is back up");
				}
			$out = &run_on_command($serv, $serv->{'onup'});
			if ($notify{'email'}) {
				$thisemail .= "Monitor on $host for '$serv->{'desc'}'\n".
				 "has detected that the service has gone back up at $now\n";
				if ($out) {
					$thisemail .= $out;
					}
				$thisemail .= "\n";
				$ecount++;
				}
			}
		elsif ($up == -1) {
			$subj = "$serv->{'desc'} uninstalled on $host";
			if ($notify{'pager'}) {
				$pager_msg .= "$host: \"$serv->{'desc'}\" uninstalled $now! ";
				}
			if ($notify{'snmp'}) {
				push(@snmp_msg, "$host: $serv->{'desc'} uninstalled");
				}
			if ($notify{'email'}) {
				$thisemail .= "Monitor on $host for '$serv->{'desc'}'\n".
				 "has detected that the service is uninstalled at $now\n\n";
				$ecount++;
				}
			}
		elsif ($up == -2) {
			$subj = "$serv->{'desc'} Webmin down on $host";
			if ($notify{'pager'}) {
				$pager_msg .= "$host: \"$serv->{'desc'}\" Webmin down $now! ";
				}
			if ($notify{'snmp'}) {
				push(@snmp_msg, "$host: $serv->{'desc'} Webmin down");
				}
			if ($notify{'email'}) {
				$thisemail .= "Monitor on $host for '$serv->{'desc'}'\n".
				 "has detected that Webmin is down at $now\n\n";
				$ecount++;
				}
			}
		elsif ($up == -3) {
			$subj = "$serv->{'desc'} timed out on $host";
			if ($notify{'pager'}) {
				$pager_msg .= "$host: \"$serv->{'desc'}\" timed out $now! ";
				}
			if ($notify{'snmp'}) {
				push(@snmp_msg, "$host: $serv->{'desc'} timed out");
				}
			if ($notify{'email'}) {
				$thisemail .= "Monitor on $host for '$serv->{'desc'}'\n".
				 "has timed out at $now\n\n";
				$ecount++;
				}
			}
		}
	elsif ($warn == 2 && $up == 0) {
		# Service is down now
		$subj = "$serv->{'desc'} down on $host";
		if ($notify{'pager'}) {
			$pager_msg .= "$host: \"$serv->{'desc'}\" is down $now! ";
			}
		if ($notify{'snmp'}) {
			push(@snmp_msg, "$host: $serv->{'desc'}");
			}
		$out = &run_on_command($serv, $serv->{'ondown'});
		if ($notify{'email'}) {
			$thisemail .= "Monitor on $host for '$serv->{'desc'}'\n".
			 "has detected that the service is down at $now\n";
			if ($out) {
				$thisemail .= $out;
				}
			$thisemail .= "\n";
			$ecount++;
			}
		}
	$oldstatus{$serv->{'id'}} = $up;

	if ($serv->{'email'} && $thisemail) {
		# If this service has an extra email address specified,
		# send to it
		&send_status_email($thisemail,
		  $config{'subject_mode'} ? $subj : "Service monitor : $subj",
		  $serv->{'email'});
		}

	$email .= $thisemail;
	if ($config{'sched_single'} && $email) {
		# Force the sending of one email and page per report
		&send_status_email($email,
		  $config{'subject_mode'} ? $subj : "Service monitor : $subj",
		  $config{'sched_email'});
		undef($email);
		if ($pager_msg) {
			&send_status_pager($pager_msg);
			}
		undef($pager_msg);
		}

	# If any SNMP messages are defined, send them
	if (@snmp_msg) {
		&send_status_trap(@snmp_msg);
		undef(@snmp_msg);
		}
	}

# Close oldstatus and fails files
&write_file($oldstatus_file, \%oldstatus);
&unlock_file($oldstatus_file);
&write_file($fails_file, \%fails);
&unlock_file($fails_file);

# Send the email and page with all messages, if necessary
if ($ecount && !$config{'sched_single'}) {
	&send_status_email($email,
			   $config{'subject_mode'} ? "Service monitor" :
			   $ecount == 1 ? "Service monitor : $subj" :
			   	          "Service monitor : $ecount services",
			   $config{'sched_email'});
	}
if ($pager_msg && !$config{'sched_single'}) {
	&send_status_pager($pager_msg);
	}

# send_status_email(text, subject, email-to)
sub send_status_email
{
return if (!$_[2]);
&foreign_require("mailboxes", "mailboxes-lib.pl");

# Construct and send the email
local $from = $config{'sched_from'} ? $config{'sched_from'}
				    : &mailboxes::get_from_address();
local $mail = { 'headers' =>
		[ [ 'From', $from ],
		  [ 'To', $_[2] ],
		  [ 'Subject', $_[1] ] ],
		'attach' =>
		[ { 'headers' => [ [ 'Content-type', 'text/plain' ] ],
		    'data' => &entities_to_ascii($_[0]) } ] };
&mailboxes::send_mail($mail, undef, 0, 0, $config{'sched_smtp'});
}

# send_status_pager(text)
# Send some message with the pager command, if configured
sub send_status_pager
{
return if (!$config{'sched_pager'});
return if (!&has_command($config{'pager_cmd'}));
system("$config{'pager_cmd'} ".quotemeta($config{'sched_pager'})." ".
       quotemeta($_[0])." >/dev/null 2>&1 </dev/null");
}

# send_status_trap(msg, ...)
# Send an SNMP trap for some message, if configured
sub send_status_trap
{
return if (!$config{'snmp_server'});

# Connect to SNMP server
eval "use Net::SNMP qw(OCTET_STRING)";
local ($session, $error) = Net::SNMP->session(
	"-hostname" => $config{'snmp_server'},
	"-port" => 162,
	"-version" => $config{'snmp_version'},
	"-community" => $config{'snmp_community'},
	);
if ($error) {
	print STDERR "SNMP connect failed : $error\n";
	return;
	}

# Send off a trap
local $rv;
local (@oids, $m);
foreach $m (@_) {
	local $oid = $config{'snmp_trap'};
	push(@oids, $oid, 4, $m);
	}
if ($config{'snmp_version'} == 1) {
	$rv = $session->trap(
		"-varbindlist" => \@oids);
	}
elsif ($config{'snmp_version'} >= 2) {
	@oids = ( "1.3.6.1.2.1.1.3.0", 67, 0,
		  "1.3.6.1.6.3.1.1.4.1.0", 6, $oids[0],
		  @oids );
	$rv = $session->snmpv2_trap(
		"-varbindlist" => \@oids);
	}
if (!$rv) {
	print STDERR "trap failed! : ",$session->error(),"\n";
	}
}

# run_on_command(&serv, command)
sub run_on_command
{
return undef if (!$_[1]);
local $out;
if ($_[0]->{'runon'} && $_[0]->{'remote'}) {
	# Run on the remote host
	local $cmd = $_[1];
	$cmd =~ s/\\/\\\\/g;
	$cmd =~ s/'/\\'/g;
	if ($config{'output'}) {
		$out = &remote_eval($_[0]->{'remote'}, "status",
			     "`($cmd) 2>&1 </dev/null`");
		return "Running $_[1] on $_[0]->{'remote'} ..\n".
		       $out;
		}
	else {
		&remote_eval($_[0]->{'remote'}, "status",
			     "system('($cmd) >/dev/null 2>&1 </dev/null')");
		return "Running $_[1] on $_[0]->{'remote'}\n";
		}
	}
else {
	# Just run locally
	if ($config{'output'}) {
		$out = `($_[1]) 2>&1 </dev/null`;
		return "Running $_[1] ..\n".
		       $out;
		}
	else {
		system("($_[1]) >/dev/null 2>&1 </dev/null");
		return "Running $_[1] ..\n";
		}
	}
}

