#!/opt/vdops/bin/perl

# This script queries the Environmental Monitoring function contained in most, 
# though not all, Cisco hardware and produces a report

# V      Who       When        What
# ---------------------------------------------------------------------------
# 1.8.0  skendric  2011-02-21  Upgrade to Netops 1.4.0
# 1.7.5  skendric  2011-01-03  Log bad values for entSensorStatus
# 1.7.4  skendric  2010-12-30  Add @insane to report
# 1.7.3  skendric  12-17-2010  Futz with owner/owner_backup
# 1.7.2  skendric  12-13-2010  Move write_log to NetopsTools.pm
# 1.7.1  skendric  08-26-2010  Add logging
# 1.7.0  skendric  03-30-2010  Upgrade to perl 5.10.1
# 1.6.3  skendric  04-19-2009  Distinguish between silent and unresponsive
# 1.6.2  skendric  03-20-2009  Add @down_for_maintenance
# 1.6.1  skendric  12-07-2007  Handle intermittently available devices
# 1.6.0  skendric  11-21-2007  Move alarm subroutines to CiscoTools
# 1.5.9  skendric  09-18-2007  Handle devices with 'notPresent' thermometers
# 1.5.8  skendric  07-26-2007  Handle devices which don't support EnvMonPs
# 1.5.7  skendric  07-16-2007  Fiddle with report format
# 1.5.6  skendric  04-10-2007  Analyze power supplies using 
#                              CISCO-ENTITY-FRU-CONTROL-MIB
# 1.5.5  skendric  03-21-2007  Stylistic mods 
# 1.5.4  skendric  02-11-2007  Add support for cefcFanTrayOperStatus
# 1.5.3  skendric  01-14-2007  Move ciscoEnvMonState to SNMPTools.pm
# 1.5.2  skendric  12-03-2006  Replace Object Values with OIDs
# 1.5.1  skendric  11-05-2005  Upgrade to new WI::VDOPS module structure
# 1.5.0  skendric  05-09-2005  Support Netops.pm-1.2
# 1.4.3  skendric  02-24-2005  Rename to cisco-alarm
# 1.4.2  skendric  02-24-2005  Add detail to alarm notification e-mail
# 1.4.1  skendric  02-18-2005  Add notFunctioning to envMonState
# 1.4.0  skendric  01-11-2005  Add FanState
# 1.3.1  skendric  08-09-2004  Fiddled with report width
# 1.3.0  skendric  05-09-2004  Migrate common functions to Netops.pm
# 1.2.0  skendric  04-30-2004  Enhance command-line options
# 1.1.2  skendric  11-16-2003  Use Net::Ping::External
# 1.1.1  skendric  08-10-2003  Minor bug fixes
# 1.1.0  skendric  03-28-2003  Numerous minor updates
# 1.0.0  skendric  03-03-2003  First version
#
#
# Author:  Stuart Kendrick, sbk {put at sign here} skendric {put dot here} com
#
# Source:  http://www.skendric.com/device
#
# This software is available under the GNU GENERAL PUBLIC LICENSE, see
# http://www.fsf.org/licenses/gpl.html
#

#
# This script takes the following approach:
#     -Parses the hosts table for a list of targets (or accepts a command-
#      line list)
#     -Queries a bunch of CISCO-ENVMON-MIB specific variables
#     -Produces a report
#
#
# Requirements:
#      -The target(s) must be pingable
#
#      -The following MIB modules stashed in /opt/vdops/share/snmp/mibs,
#       or wherever it is that you store MIB modules:
#       CISCO-PRODUCTS-MIB.my
#
#     -PERL modules:  the WI::Netops collection
#
#
# Assumptions:
#
#
# Tested on:
#     -perl-5.12.2
#     -net-snmp-5.6
#
#
# Instructions:
#     -Customize the script for your site:  find the 'user-configurable
#      variables' section and modify as appropriate
#     -Try it out
#
#
# Caveats:
#
#
# Known Bugs:
#
#
# To do:
#     -Add support for SNMPv3
#

# Begin script

# Load modules
use strict;
use warnings;
use feature 'say';
use feature 'switch';
use Carp qw(carp cluck croak confess);
use Data::Dumper;
use English qw( -no_match_vars );
use Getopt::Std;
use List::MoreUtils qw(any);
use WI::Netops::CiscoTools 1.4.3;
use WI::Netops::HostTools 1.0.4;
use WI::Netops::NetopsTools 2.2.3;
use WI::Netops::NetopsData 1.4.0;
use WI::Netops::PingTools 1.1.7;
use WI::Netops::SNMPTools 1.5.3;
use WI::Netops::Utilities 1.4.4;

# Declare global variables.  All hashes keyed by target
my %alarms;           # Hash of array refs, listing problems
my %fan_state;        # State of most degraded fan, ciscoEnvMonFanState
my $log_file;         # Location of log file
my %module_state;     # State of most degraded module, cefcModuleOperStatus
my %ps_state;         # State of most degraded power supply, 
                      # ciscoEnvMonSupplyState
my %sensor_state;     # worst of entSensorStatus
my %temp_state;       # State of most degraded thermometer, 
                      # ciscoEnvMonTemperatureState
my %tray_state;       # State of most degraded fan tray, cefcFanTrayOperStatus
my %volt_state;       # State of most degraded voltage sensor, 
                      # ciscoEnvMonVoltageState

# Define global variables
$program_name = 'cisco-alarm';
$usage = 'Usage:  cisco-alarm -s {yes|no} [-d {integer}] [-r] [-a | -e {expr} | -f {filename} | target1 target2 target3 ...]'; 
$version = '1.8.0';


# Grab arguments
getopts('ad:e:f:rs:', \%option); 
@target = @ARGV;

# Set mode
if      ($option{r})            { $mode = 'report'      }
elsif   (-t STDIN)              { $mode = 'interactive' }
else                            { $mode = 'batch'       }


### Begin Main Program ###############################################
{

  check_args();                   # Check arguments

  read_config();                  # Read Netops config file

  compile_mibs();                 # Compile MIB files

  build_target();                 # Populate @target
  
  target_check();                 # Look for errors in @target

  basic_info();                   # Gather information

  sanity_check();                 # Check for error conditions

  do_the_work();                  # Do it

  identify_alarms();              # Count devices with alarms

  write_alarm_log();              # Record issues

  print_report();                 # Print report

  notify_staff();                 # Mail report
}

##### End Main Program #################################################



########################################################################
# Query variables
########################################################################
sub do_the_work {

  # Debug trace
  trace_location('begin') if $debug;

  # Notify operator
  print_it('Querying targets...');
  unless ($dome) {
    sleep $short;
    return 1;
  }

  # Loop through the list of targets
  for my $target (@target) {
    my (@status, $vb);
    say "Processing $target" if $debug;

    # Fans
    $fan_state{$target}   = acquire_fan_alarm_state($target);
    $fan_state{$target} //= $QUERY;
    unless ($fan_state{$target} eq 'ok' or $fan_state{$target} eq $DASH) {
      log_it("For $target, fan state = $fan_state{$target}");
      $alarm_count{$target}++;
      push @{$alarms{$target}}, 'fan';
    }

    # Power supplies
    $ps_state{$target}   = acquire_power_alarm_state($target);
    $ps_state{$target} //= $QUERY;
    unless ($ps_state{$target} eq 'ok' or $ps_state{$target} eq $DASH) {
      log_it("For $target, ps state = $ps_state{$target}");
      $alarm_count{$target}++;
      push @{$alarms{$target}}, 'PS';
    }

    # Thermometers
    $temp_state{$target}   = acquire_temperature_alarm_state($target);
    $temp_state{$target} //= $QUERY;
    unless ($temp_state{$target} eq 'ok' or $temp_state{$target} eq $DASH) {
      log_it("For $target, temp state = $temp_state{$target}");
      $alarm_count{$target}++;
      push @{$alarms{$target}}, 'temp';
    }

    # Voltage probes
    $volt_state{$target}   = acquire_voltage_alarm_state($target);
    $volt_state{$target} //= $QUERY;
    unless ($volt_state{$target} eq 'ok' or $volt_state{$target} eq $DASH) {
      log_it("For $target, volt state = $volt_state{$target}");
      $alarm_count{$target}++;
      push @{$alarms{$target}}, 'volt';
    }

    # Fan trays
    $tray_state{$target}   = acquire_tray_alarm_state($target);
    $tray_state{$target} //= $QUERY;
    unless ($tray_state{$target} eq 'ok' or $tray_state{$target} eq $DASH) {
      log_it("For $target, tray state = $tray_state{$target}");
      $alarm_count{$target}++;
      push @{$alarms{$target}}, 'fan';
    }

    # Modules
    $module_state{$target} = acquire_module_alarm_state($target);
    $module_state{$target} //= $QUERY;
    unless ($module_state{$target} eq 'ok' or $module_state{$target} eq $DASH) {
      given ($box{$target}) {
        when (/cat65/) { my $e = 'Do nothing; bug in IOS' }
        default        {
          log_it("For $target, module state = $module_state{$target}");
          $alarm_count{$target}++;
          push @{$alarms{$target}}, 'module';
        }
      }
    }

    # Sensor status
    $vb = snmpWalk( {host => $target, oid => 'entSensorStatus'} );
    for my $varbind (@$vb) {
      push @status, $varbind->{val};
    }
    if (any {$_ eq 'ok'            } @status) { $sensor_state{$target} = 'ok'  }
    if (any {$_ eq 'unavailable'   } @status) { $sensor_state{$target} = 'na'  }
    if (any {$_ eq 'nonoperational'} @status) { $sensor_state{$target} = 'bad' }
    $sensor_state{$target} //= $QUERY;
    if ($sensor_state{$target} eq 'bad') {
      unless ($box{$target} eq 'cat6504') {  # Bug in Cat6504
        log_it("Sensor state on $target is $sensor_state{$target}");
        $alarm_count{$target}++;
        push @{$alarms{$target}}, 'sensor';
      }
    }

    # Entertain operator
    print $BANG if $mode eq 'interactive';

  }

  # Debug info
  if ($debug > 2) {
    for my $target (@target) {
      if (defined $alarm_count{$target} and $alarm_count{$target} > 0) {
        say "alarm_count{$target} = $alarm_count{$target}";
      }
    }
  }

  # Make things look pretty
  say "\n" if $mode eq 'interactive';

  # Debug trace
  trace_location('end') if $debug;

  return 1;  
}




########################################################################
# Tell the operator what I discovered
########################################################################
sub print_report {
  my $handle;
  my $total = @target;
  my $now = get_now();

  # If we are running in test mode, skip this routine
  unless ($dome) {
    print_it("Running in test mode, cannot print a meaningful report\n");
    return 1;
  }

  # Debug trace
  trace_location('begin') if $debug;

  # Direct output to screen or to file
  if ($mode eq 'interactive') {
    $handle = *STDOUT;
  }
  else {
    open $handle, '>', $report_file or die "Cannot open $report_file: $!\n";
  }


print {$handle} <<EOF;
# Title:           $report_subject
#
# Institution:     $institution
#
# Date of Report:  $now
#
# Description:     This report lists the status of Cisco hardware components
#
#                  Look through syslog for the details of alarms:
#                       grep $program_name /var/log/syslog
#
# Active:          $total
#
# Alarm Count:     $shit_happens
#
# In Alarm:        @alarm_list
# 
# Questions:       If you have questions or comments regarding this
#                  report, please mail them to $report_queries.
#
# Notes:           This report lists power supply, fan, temperature, 
#                  voltage, fan tray, and module status as applicable 
#                  to each flavor of hardware.  If a device does not 
#                  support the appropriate MIB variable, the report 
#                  lists a '-' instead of a status descriptor.  If 
#                  the device reports more than one problematic entity 
#                  (e.g. power supply, fan, thermometer, voltage probe, 
#                  fan tray, or module), then the report only lists 
#                  the status of one of the problematic entities.
#                  This report relies on CISCO-ENV-MIB and 
#                  CISCO-ENTITY-FRU-CONTROL-MIB.
#
# Most columns = ok, warning, critical, shutdown, notPresent, notFunctioning
#
# PS           = Under some conditions, Power Supply can appear as: 
#                offAdmin, offDenied, offEnvPower, offEnvTemp, offEnvFan, 
#                failed, onButFanFail, offCooling, offConnectorRating
#
# Fan Tray     = unknown, ok, down, warning
#
# Module       = unknown, ok, disabled, okButDiagFailed, boot, selfTest,
#                failed, missing, mismatchWithParent, mismatchConfig, 
#                diagFailed, dormant, outOfServiceAdmin, outOfServiceEnvTemp, 
#                poweredDown, poweredUp, powerDenied, powerCycled, 
#                okButPowerOverWarning, okButPowerOverCritical,
#                syncInProgress
#
target             PS           Fans  Temp    Voltage Tray    Module
------------------ ------------ ----- ------- ------- ------- -----------------
EOF
  for my $target (@target) {
    printf {$handle} 
      "%-18s %-12.12s %-5.5s %-7.7s %-7.7s %-7.7s %-18.18s\n",
      $target, $ps_state{$target}, $fan_state{$target}, $temp_state{$target}, 
      $volt_state{$target}, $tray_state{$target}, $module_state{$target};
  }

  # Add silent devices to the report
  for my $silent (sort @silent) {
    printf {$handle} "%-15s  Not answering pings\n", $silent;
  }

  # Add unresponsive devices to the report
  for my $unresponsive (sort @unresponsive) {
    printf {$handle} "%-15s  Not answering SNMP GETs\n", $unresponsive;
  }

  # Add insane devices to the report
  for my $insane (sort @insane) {
    printf {$handle} "%-15s  Insane\n", $insane;
  }

  # Clean up
  unless ($handle =~ /STDOUT/) {
    close $handle or warn "Cannot close $report_file: $!\n";
  }

  # Debug trace
  trace_location('end') if $debug;

  return 1;
}




########################################################################
# Sanity check
########################################################################
sub sanity_check {
  my $cefc_fru;
  my $envmon;
  my @remove;

  # Debug trace
  trace_location('begin') if $debug;

  # Notify operator
  print_it('Sanity check...');

  # Loop through targets, looking for problems
  for my $target (@target) {

    # Identify presence of Cisco Environmental Monitoring function
    $envmon   = snmpGet({host => $target, oid => 'ciscoEnvMonPresent.0'}     );
    $cefc_fru = snmpWalk({host => $target, oid => 'cefcFRUPowerAdminStatus'});
    unless (defined $envmon or @$cefc_fru > 0) {
      say "\n$target does not support CISCO-ENVMON-MIB or CISCO-ENTITY-FRU-CONTROL-MIB, ignoring" if $debug;
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
    }
    else {
      print $BANG if $mode eq 'interactive';
    }

  }

  # Remove entries which failed checks
  prune_basic(@remove);
  #@insane = @remove;

  # Make things look pretty
  say "\n" if $mode eq 'interactive';

  # Debug trace
  trace_location('end') if $debug;

  return 1;
}




########################################################################
# Output help
########################################################################
sub HELP_MESSAGE {
  print <<EOF;

  See $netops_global_config (and $netops_alarm_config if this is an
  xxxx-alarm script) for configuration options

  $usage

    -s asks the question:  are you serious?  If you answer 'no', then 
       the script will run in demo mode, making no changes

    -d specifies debug level (typically 0-9)

    -r determines whether or not the script writes its report to disk

    -a instructs the script to filter the hosts table using its internally
       defined suffixes variable to produce a list of targets

    -e requires a regex as its argument, with which the script filters
       the hosts table to produce a list of targets.  Enclose the regex in 
       quotes to protect special characters from the shell

    -f points the script to a file containing the list of targets

    target1 target2 target3 ... is the command-line list of targets you
      have specified, in lieu of the -a, -e, or -f options

EOF

  return 1;
}




########################################################################
# Output version
########################################################################
sub VERSION_MESSAGE {
  say "$program_name v$version";
  return 1;
}