#!/opt/vdops/bin/perl

# This script queries Cisco Communications Manager variables, alarms if it 
# doesn't like what it sees, and produces a report

# V      Who       When        What
# ---------------------------------------------------------------------------
# 1.2.0  skendric  2011-02-21  Upgrade to Netops 1.4.0
# 1.1.1  skendric  2010-12-30  Add @insane to report
# 1.1.0  skendric  2010-12-17  Add thresholding to registered stuff
# 1.0.2  skendric  2010-12-15  Require support for CISCO-CCM-MIB
# 1.0.1  skendric  2010-12-10  Handle non-responsive stations
# 1.0.0  skendric  2010-12-08  First Version
#
#
# Author:  Stuart Kendrick, sbk {put at sign here} skendric {put dot here} com
#
# Source:  http://www.skendric.com/device
#
# This software is available under the GNU GENERAL PUBLIC LICENSE, see
# http://www.fsf.org/licenses/gpl.html
#

#
# This script takes the following approach:
#     -Parses the hosts table for a list of targets (or accepts a command-
#      line list)
#     -Queries a bunch of CISCO-CCM-MIB specific variables
#     -Produces a report
#
#
# Requirements:
#      -The target(s) must be pingable
#
#      -The following MIB modules stashed in /opt/vdops/share/snmp/mibs,
#       or wherever it is that you store MIB modules:
#       CISCO-PRODUCTS-MIB.my
#
#     -PERL modules:  the WI::Netops collection
#
#
# Assumptions:
#
#
# Tested on:
#     -perl-5.12.2
#     -net-snmp-5.6
#
#
# Instructions:
#     -Customize the script for your site:  find the 'user-configurable
#      variables' section and modify as appropriate
#     -Try it out
#
#
# Caveats:
#
#
# Known Bugs:
#
#
# To do:
#     -Add support for SNMPv3
#

# Begin script

# Load modules
use strict;
use warnings;
use feature 'say';
use feature 'switch';
use Carp qw(carp cluck croak confess);
use Data::Dumper;
use English qw( -no_match_vars );
use Getopt::Std;
use List::MoreUtils qw(first_index);
use Sys::Hostname;
use WI::Netops::CiscoTools 1.4.3;
use WI::Netops::HostTools 1.0.4;
use WI::Netops::NetopsTools 2.2.3;
use WI::Netops::NetopsData 1.4.0;
use WI::Netops::PingTools 1.1.7;
use WI::Netops::SNMPTools 1.5.3;
use WI::Netops::Utilities 1.4.4;

# Declare global variables.  All hashes keyed by target
my %ccm_alarm_enable;            # Roll-up of various AlarmEnable variables
my %ccm_descr;                   # ccmDescription
my $ccm_log_file;                # Where we write the count of registered phones
my %ccm_status;                  # ccmStatus
my %partially_registered_phones; # ccmRegisteredPhones
my $registered_gateways;         # Sum of ccmRegisteredGateways
my $registered_media_devices;    # Sum of ccmRegisteredMediaDevices
my $registered_phones;           # Sum of ccmRegisteredPhones
my %registered_gateways;         # ccmRegisteredGateways
my %registered_media_devices;    # ccmRegisteredMediaDevices
my %registered_phones;           # ccmRegisteredPhones
my %rejected_gateways;           # ccmRejectedGateways
my %rejected_media_devices;      # ccmRejectedMediaDevices
my %rejected_phones;             # ccmRejectedPhones
my $threshold;                   # If $registered_xxx / $yesterday_xxx is less 
                                 # than this threshold, whine
my %unregistered_gateways;       # ccmUnregisteredGateways
my %unregistered_media_devices;  # ccmUnregisteredMediaDevices
my %unregistered_phones;         # ccmUnregisteredPhones
my $yesterday_gateways;          # Minimum number of gateways
my $yesterday_media_devices;     # Minimum number of media_devices
my $yesterday_phones;            # Minimum number of phones

# Define global variables
$program_name = 'ccm-alarm';
$usage = 'Usage:  ccm-alarm -s {yes|no} [-d {integer}] [-r] [-a | -e {expr} | -f {filename} | target1 target2 target3 ...]'; 
$version = '1.2.0';

# Thresholds
$threshold = .8;

# Geport stuff
$ccm_log_file      = '/home/netops/logs/ccm-registered-phones.log'; 

getopts('ad:e:f:rs:', \%option); 
@target = @ARGV;

# Set mode
if      ($option{r})            { $mode = 'report'      }
elsif   (-t STDIN)              { $mode = 'interactive' }
else                            { $mode = 'batch'       }


### Begin Main Program ###############################################

{
  check_args();                   # Check arguments

  read_config();                  # Read Netops config file

  compile_mibs();                 # Compile MIB files

  build_target();                 # Populate @target
  
  target_check();                 # Look for errors in @target

  basic_info();                   # Gather information

  sanity_check();                 # Sanity check

  info_before();                  # Gather more information

  do_the_work();                  # Do it

  consider_thresholds();          # Compare counts to thresholds

  identify_alarms();              # Count devices with alarms

  write_alarm_log();              # Record issues

  write_log();                    # Record stats

  print_report();                 # Print report

  notify_staff();                 # Mail report
}

##### End Main Program #################################################



########################################################################
# Compare counts to thresholds
########################################################################
sub consider_thresholds {
  my $publisher;

  # Debug trace
  trace_location('begin') if $debug;

  # Identify Publisher
  # Sum counters
  for my $target (@target) {
    $registered_gateways      += $registered_gateways{$target};
    $registered_media_devices += $registered_media_devices{$target};
    $registered_phones        += $registered_phones{$target};
    $publisher = $target if $ccm_descr{$target} =~ /Publisher/;
  }

  # Log what we see
  log_it("For cluster $publisher, I see $registered_gateways registered gateways");
  log_it("For cluster $publisher, I see $registered_media_devices registered media devices");
  log_it("For cluster $publisher, I see $registered_phones registered phones");


  # Compare to thresholds
  if ($registered_gateways / $yesterday_gateways < $threshold) {
    log_it("For cluster $publisher, insufficient gateways -- Yesterday:  $yesterday_gateways, Today:  $registered_gateways");
    $alarm_count{$publisher}++;
    push @{$alarms{$publisher}}, "Insufficient gateways: $registered_gateways";
  }
  if ($registered_media_devices / $yesterday_media_devices < $threshold) {
    log_it("For cluster $publisher, insufficient media devices -- Yesterday: $yesterday_media_devices, Today: $registered_media_devices");
    $alarm_count{$publisher}++;
    push @{$alarms{$publisher}}, "Insufficient media devices: $registered_media_devices";
  }
  if ($registered_phones / $yesterday_phones < $threshold) {
    log_it("For cluster $publisher, insufficient phones -- Yesterday:  $yesterday_phones, Today: $registered_phones");
    $alarm_count{$publisher}++;
    push @{$alarms{$publisher}}, "Insufficient phones: $registered_phones";
  }

  # Debug trace
  trace_location('end') if $debug;

  return 1;
}




########################################################################
# Query variables
########################################################################
sub do_the_work {

  # Debug trace
  trace_location('begin') if $debug;

  # Notify operator
  print_it('Querying targets...');
  unless ($dome) {
    sleep $short;
    return 1;
  }

  # Loop through the list of targets
  for my $target (@target) {
    my $val;
    say "Processing $target" if $debug;

    # ccmStatus
    $val = acquire_ccm_status($target);
    $val //= $QUERY;
    unless ($val eq 'up') {
      log_it("For $target, ccmStatus = $val");
      push @{$alarms{$target}}, "ccmStatus is $val";
      $alarm_count{$target}++;
    }
    $ccm_status{$target} = $val;

    # ccmDescrption
    $ccm_descr{$target} = acquire_ccm_descr($target);
    $ccm_descr{$target} //= $QUERY;

    # Roll-up of AlarmEnable variables
    $ccm_alarm_enable{$target} = acquire_ccm_alarm_enable($target);
    $ccm_alarm_enable{$target} //= $QUERY;
    unless ($ccm_alarm_enable{$target} eq 'yes') {
      log_it("For $target, ccmXXXAlarmEnable = $ccm_alarm_enable{$target}");
      push @{$alarms{$target}}, 'Alarms are disabled';
      $alarm_count{$target}++;
    }


    # Count registered gateways
    $val = snmpGet( {host => $target, oid => 'ccmRegisteredGateways.0'} );
    if (defined $val) {
      given ($ccm_descr{$target}) {
        when (/Publisher/) {
          if ($val > 0) {
            log_it("For $target, ccmRegisteredGateways: $val");
            push @{$alarms{$target}}, "Registered gateways: $val";
            $alarm_count{$target}++;
          }
        }
        when (/Subscriber/) {
          unless ($val > 0) {
            log_it("For $target, ccmRegisteredGateways: 0");
            push @{$alarms{$target}}, 'Registered gateways: 0';
            $alarm_count{$target}++;
          }
        }
      }
    } 
    $val //= $QUERY;
    $registered_gateways{$target} = $val;

    # Count unregistered gateways
    $val = snmpGet( {host => $target, oid => 'ccmUnregisteredGateways.0'} );
    if (defined $val) {
      log_it("For $target, ccmUnregisteredGateways: $val") if $val > 0;
    }
    $val //= $QUERY;
    $unregistered_gateways{$target} = $val;

    # Count rejected gateways
    $val = snmpGet( {host => $target, oid => 'ccmRejectedGateways.0'} );
    if (defined $val) {
      log_it("For $target, ccmRejectedGateways: $val") if $val > 0;
    }
    $val //= $QUERY;
    $rejected_gateways{$target} = $val;


    # Count registered media devices
    $val = snmpGet( {host => $target, oid => 'ccmRegisteredMediaDevices.0'} );
    if (defined $val) {
      given ($ccm_descr{$target}) {
        when (/Publisher/) {
          if ($val > 0) {
            log_it("For $target, ccmRegisteredMediaDevices: $val");
            push @{$alarms{$target}}, "Registered gateways: $val";
            $alarm_count{$target}++;
          }
        }
        when (/Subscriber/) {
          unless ($val > 0) {
            log_it("For $target, ccmRegisteredMediaDevices: 0");
            push @{$alarms{$target}}, 'Registered media devices: 0';
            $alarm_count{$target}++;
          }
        }
      }
    }
    $val //= $QUERY;
    $registered_media_devices{$target} = $val;

    # Count unregistered media devices
    $val = snmpGet( {host => $target, oid => 'ccmUnregisteredMediaDevices.0'} );
    if (defined $val) {
      log_it("For $target, ccmUnregisteredMediaDevices: $val") if $val > 0;
    }
    $val //= $QUERY;
    $unregistered_media_devices{$target} = $val;

    # Count rejected media devices
    $val = snmpGet( {host => $target, oid => 'ccmRejectedMediaDevices.0'} );
    if (defined $val) {
      log_it("For $target, ccmRejectedMediaDevices: $val") if $val > 0;
    }
    $val //= $QUERY;
    $rejected_media_devices{$target} = $val;


    # Count registered phones
    $val = snmpGet( {host => $target, oid => 'ccmRegisteredPhones.0'} );
    if (defined $val) {
      given ($ccm_descr{$target}) {
        when (/Publisher/) {
          if ($val > 0) {
            log_it("For $target, ccmRegisteredPhones: $val");
            push @{$alarms{$target}}, "Registered phones: $val";
            $alarm_count{$target}++;
          }
        }
        when (/Subscriber/) {
          unless ($val > 0) {
            log_it("For $target, ccmRegisteredPhones: 0");
            push @{$alarms{$target}}, 'Registered phones: 0';
            $alarm_count{$target}++;
          }
        }
      }
    }
    $val //= $QUERY;
    $registered_phones{$target} = $val;

    # Count unregistered phones
    $val = snmpGet( {host => $target, oid => 'ccmUnregisteredPhones.0'} );
    if (defined $val) {
      log_it("For $target, ccmUnregisteredPhones: $val") if $val > 0;
    }
    $val //= $QUERY;
    $unregistered_phones{$target} = $val;

    # Count rejected phones
    $val = snmpGet( {host => $target, oid => 'ccmRejectedPhones.0'} );
    if (defined $val) {
      log_it("For $target, ccmRejectedPhones: $val") if $val > 0;
    }
    $val //= $QUERY;
    $rejected_phones{$target} = $val;


    # Count partially registered phones
    $val = snmpGet({host => $target, oid => 'ccmPartiallyRegisteredPhones.0'});
    if (defined $val) {
      if ($val > 0) {
        log_it("For $target, ccmPartiallyRegisteredPhones: $val");
        push @{$alarms{$target}}, "Partially registered phones: $val";
        $alarm_count{$target}++;
      }
    }
    $val //= $QUERY;
    $partially_registered_phones{$target} = $val;


    # Entertain operator
    print $BANG if $mode eq 'interactive';
  }

  # Debug info
  if ($debug > 2) {
    for my $target (@target) {
      if (defined $alarm_count{$target} and $alarm_count{$target} > 0) {
        say "alarm_count{$target} = $alarm_count{$target}";
      }
    }
  }

  # Make things look pretty
  say "\n" if $mode eq 'interactive';

  # Debug trace
  trace_location('end') if $debug;

  return 1;  
}




#########################################################################
# Gather additional information
########################################################################
sub info_before {

  # Debug trace
  trace_location('begin') if $debug;

  # Notify operator
  print_it('Gathering more information...');

  # Open yesterday's report
  open my $file, '<', $report_file or warn "Cannot open $report_file:  $!  -- estimating previous counts";

  # Dig out yesterday's counts
  if (fileno $file) {
    LINE:
    while (my $line = <$file>) {
      given ($line) {
        when (/Registered Phones/) {
          ($yesterday_phones) = ($line =~ /Registered Phones\s+(\d+)/);
        }
        when (/Registered Gateways/) {
          ($yesterday_gateways) = ($line =~ /Registered Gateways\s+(\d+)/);
        }
        when (/Registered Media Devices/) {
          ($yesterday_media_devices) = ($line =~ /Registered Media Devices\s+(\d+)/);
          last LINE;
        }
      }
    }
    close $file or warn "Cannot close $report_file: $!";
  }
  
  # Assign values to yesterday's counts, if still empty
  $yesterday_phones        //= 1000;
  $yesterday_gateways      //= 50;
  $yesterday_media_devices //= 5;

  # Debug info
  if ($debug) {
    say "Yesterday's Phones = $yesterday_phones";
    say "Yesterday's Gateways = $yesterday_gateways";
    say "Yesterday's Media Devices = $yesterday_media_devices";
  }

  # Debug trace
  trace_location('end') if $debug;

  return 1;
}




#########################################################################
# Sanity check
########################################################################
sub sanity_check {
  my @remove;

  # Debug trace
  trace_location('begin') if $debug;

  # Notify operator
  print_it('Sanity check...');

  # Loop through targets, looking for problems
  for my $target (@target) {
    my $reg_phones;

    # Identify presence of Cisco Call Manager MIB
    $reg_phones = snmpGet({host => $target, oid => 'ccmRegisteredPhones.0'} );
    unless (defined $reg_phones) {
      say "\n$target does not support CISCO-CCM-MIB, ignoring" if $debug;
      push @remove, $target; 
      print $DOT if $mode eq 'interactive';
    }
    else {
      print $BANG if $mode eq 'interactive';
    }
  
  }
  
  # Save unresponsive targets
  push @unresponsive, @remove;

  # Remove entries which failed checks
  prune_basic(@remove);
  @insane = @remove;

  # Make things look pretty
  say "\n" if $mode eq 'interactive';
  
  # Debug trace
  trace_location('end') if $debug;

  return 1;
}




########################################################################
# Tell the operator what I discovered
########################################################################
sub print_report {
  my $handle;
  my $total = @target;
  my $now = get_now();

  # If we are running in test mode, skip this routine
  unless ($dome) {
    print_it("Running in test mode, cannot print a meaningful report\n");
    return 1;
  }

  # Debug trace
  trace_location('begin') if $debug;

  # Direct output to screen or to file
  if ($mode eq 'interactive') {
    $handle = *STDOUT;
  }
  else {
    open $handle, '>', $report_file or die "Cannot open $report_file: $!\n";
  }


print {$handle} <<EOF;
# Title:           $report_subject
#
# Institution:     $institution
#
# Date of Report:  $now
#
# Description:     This report lists the status of Cisco Communications Manager
#                  clusters:  Publisher + Subscribers
#
#                  Look through syslog for the details of alarms:
#                       grep $program_name /var/log/syslog
#
# Active:          $total
#
# Alarm Count:     $shit_happens
#
# In Alarm:        @alarm_list
# 
#                           Today    Yesterday
# Registered Phones         $registered_phones     $yesterday_phones
# Registered Gateways       $registered_gateways      $yesterday_gateways
# Registered Media Devices  $registered_media_devices        $yesterday_media_devices
#
# Questions:       If you have questions or comments regarding this
#                  report, please mail them to $report_queries.
#
# Status     up, down, unknown
# Type       Publisher, Subscriber
# Ena        Configured to alarm on various conditions:  yes, no
#
# RegP       Registered Phones           (want zero for Pub, non-zero for Sub)
# PaP        Partially Registered Phones (want zero)
# UnP        Unregistered Phones         (want zero)
# RejP       Rejected Phones             (want zero)
#
# RegG       Registered Gateways         (want zero for Pub, non-zero for Sub)
# UnG        Unregistered Gateways       (want zero)
# RejG       Rejected Gateways           (want zero)
#
# RegM       Registered Media Devices    (want zero for Pub, non-zero for Sub)
# UnM        Unregistered Media Devices  (want zero)
# RejM       Rejected Media Devices      (want zero)
#
target     Type Status  Ena RegP  PaP  UnP  RejP RegG UnG  RejG RegM UnM  RejM
---------- ---- ------- --- ----- ---- ---- ---- ---- ---- ---- ---- ---- ----
EOF
  for my $target (@target) {
    my $flavor;
    given ($ccm_descr{$target}) {
      when (/Publisher/)  { $flavor = 'Pub'  }
      when (/Subscriber/) { $flavor = 'Sub'  }
      default             { $flavor = $QUERY }
    }
    printf {$handle} 
      "%-10s %-4s %-7s %-3s %-5s %-4s %-4s %-4s %-4s %-4s %-4s %-4s %-4s %-4s\n",
      $target, $flavor, $ccm_status{$target}, $ccm_alarm_enable{$target},
      $registered_phones{$target}, $partially_registered_phones{$target}, 
      $unregistered_phones{$target}, $rejected_phones{$target}, 
      $registered_gateways{$target}, $unregistered_gateways{$target}, 
      $rejected_gateways{$target},
      $registered_media_devices{$target}, $unregistered_media_devices{$target}, 
      $rejected_media_devices{$target},
  }

  # Add silent devices to the report
  for my $silent (sort @silent) {
    printf {$handle} "%-15s  Not answering pings\n", $silent;
  }

  # Add unresponsive devices to the report
  for my $unresponsive (sort @unresponsive) {
    printf {$handle} "%-15s  Not answering SNMP GETs\n", $unresponsive;
  }

  # Add insane devices to the report
  for my $insane (sort @insane) {
    printf {$handle} "%-15s  Insane\n", $insane;
  }

  # Clean up
  unless ($handle =~ /STDOUT/) {
    close $handle or warn "Cannot close $report_file: $!\n";
  }

  # Debug trace
  trace_location('end') if $debug;

  return 1;
}




########################################################################
# Write summary data to a log file
########################################################################
sub write_log {
  my $date = get_date();

  # Debug trace
  trace_location('begin') if $debug;

  # Notify operator
  print_it('Writing log...');

  # Write count of registered phones
  if ($dome) {
    if (open my $log, '>>', $ccm_log_file) {
      print {$log} "$date\t$registered_phones\n";
    }
    else {
      warn "Cannot open $ccm_log_file: $!";
    }
  }

  # Make things look pretty
  say('') if $mode eq 'interactive';

  # Debug trace
  trace_location('end') if $debug;

  return 1;
}




########################################################################
# Output help
########################################################################
sub HELP_MESSAGE {
  print <<EOF;

  See $netops_global_config (and $netops_alarm_config if this is an
  xxxx-alarm script) for configuration options

  $usage

    -s asks the question:  are you serious?  If you answer 'no', then 
       the script will run in demo mode, making no changes

    -d specifies debug level (typically 0-9)

    -r determines whether or not the script writes its report to disk

    -a instructs the script to filter the hosts table using its internally
       defined suffixes variable to produce a list of targets

    -e requires a regex as its argument, with which the script filters
       the hosts table to produce a list of targets.  Enclose the regex in 
       quotes to protect special characters from the shell

    -f points the script to a file containing the list of targets

    target1 target2 target3 ... is the command-line list of targets you
      have specified, in lieu of the -a, -e, or -f options

EOF

  return 1;
}




########################################################################
# Output version
########################################################################
sub VERSION_MESSAGE {
  say "$program_name v$version";
  return 1;
}