#!/opt/vdops/bin/perl

# This script queries the CISCO-CRYPTO-ACCELERATOR-MIB, 
# CISCO-UNIFIED-FIREWALL-MIB, and CISCO-FIREWALL-MIB variables for hardware
# problems.  The Cisco ASA appliances support these MIBs (regrettably, 
# they do not support ENVMON-MIB or CISCO-ENTITY-FRU-CONTROL-MIB.)

# V      Who       When        What
# ---------------------------------------------------------------------------
# 1.2.0  skendric  2011-02-21  Upgrade to Netops 1.4.0
# 1.2.1  skendric  2010-12-30  Add @insane to report
# 1.2.0  skendric  2010-12-20  Watch memory, net, buffer, logging
# 1.1.1  skendric  2010-12-17  Futz with owner/owner_backup
# 1.1.0  skendric  2009-01-10  Upgrade to perl 5.10.1
# 1.0.5  skendric  2009-05-17  Support SNMP.pm
# 1.0.4  skendric  2009-04-19  Distinguish between silent and unresponsive
# 1.0.3  skendric  2009-03-20  Add @down_for_maintenance
# 1.0.2  skendric  2008-02-15  Spell active and standby correctly
# 1.0.1  skendric  2007-12-07  Add owner
# 1.0.0  skendric  2007-10-21  First version
#
#
# Author:  Stuart Kendrick, sbk {put at sign here} skendric {put dot here} com
#
# Source:  http://www.skendric.com/device
#
# This software is available under the GNU GENERAL PUBLIC LICENSE, see
# http://www.fsf.org/licenses/gpl.html
#

#
# This script takes the following approach:
#     -Parses the hosts table for a list of targets (or accepts a command-
#      line list)
#     -Queries a bunch of CISCO specific variables
#     -Produces a report
#
#
# Requirements:
#      -The target(s) must be pingable
#
#      -The following MIB modules stashed in /opt/vdops/share/snmp/mibs,
#       or wherever it is that you store MIB modules:
#       CISCO-PRODUCTS-MIB.my
#
#     -PERL modules:  the WI::Netops collection
#
#
# Assumptions:
#
#
# Tested on:
#     -perl-5.12.2
#     -net-snmp-5.6
#
#
# Instructions:
#     -Customize the script for your site:  find the 'user-configurable
#      variables' section and modify as appropriate
#     -Try it out
#
#
# Caveats:
#
#
# Known Bugs:
#
#
# To do:
#     -Add support for SNMPv3
#

# Begin script

# Load modules
use strict;
use warnings;
use feature 'say';
use feature 'switch';
use Carp qw(carp cluck croak confess);
use Data::Dumper;
use English qw( -no_match_vars );
use Getopt::Std;
use List::MoreUtils qw(first_index);
use WI::Netops::CiscoTools 1.4.3;
use WI::Netops::HostTools 1.0.4;
use WI::Netops::NetopsTools 2.2.3;
use WI::Netops::NetopsData 1.4.0;
use WI::Netops::PingTools 1.1.7;
use WI::Netops::SNMPTools 1.5.3;
use WI::Netops::Utilities 1.4.4;

# Declare global variables.  All hashes keyed by target
my %broken_card;         # Number of broken crypto accelerators:
                         # ccaGlobalNumNonOperAccelerators.0
my %buffer_free;         # low, mid, or high (want high)
my $buffer_thresh_low;   # Percentage of buffer utilization which triggers 
                         # a %buffer_free value of 'low'
my $buffer_thresh_high;  # Percentage of buffer utilization which triggers 
                         # a %buffer_free value of 'high'
my %declined_sess;       # Number of declined sessions (due to resource 
                         # constraints):  cufwConnGlobalNumResDeclined.0
my %logging;             # Value of cefcMIBEnableStatusNotification.0
my %memory_pool_valid;   # Value of worst case ciscoMemoryPoolValid
my %net_interface;       # Value of cfwHardwareInformation.netInterface
my %primary_detail;      # Verbose state of failover configuration:
                         # cfwHardwareStatusDetail.primaryUnit
my %primary_status;      # State of primary unit:
                         # cfwHardwareStatusValue.primaryUnit
my %secondary_detail;    # Verbose state of failover configuration:
                         # cfwHardwareStatusDetail.primaryUnit
my %secondary_status;    # State of secondary unit:
                         # cfwHardwareStatusValue.secondaryUnit


# Define global variables
$program_name = 'asa-alarm';
$usage = 'Usage:  asa-alarm -s {yes|no} [-d {integer}] [-r] [-a | -e {expr} | -f {filename} | target1 target2 target3 ...]'; 
$version = '1.3.0';

# Thresholds
$buffer_thresh_low  = .25;
$buffer_thresh_high = .75;


# Grab arguments
getopts('ad:e:f:rs:', \%option); 
@target = @ARGV;

# Set mode
if      ($option{r})            { $mode = 'report'      }
elsif   (-t STDIN)              { $mode = 'interactive' }
else                            { $mode = 'batch'       }


### Begin Main Program ###############################################
{

  check_args();                   # Check arguments

  read_config();                  # Read Netops config file

  compile_mibs();                 # Compile MIB files

  build_target();                 # Populate @target

  target_check();                 # Look for errors in @target

  basic_info();                   # Gather information

  sanity_check();                 # Check for error conditions

  do_the_work();                  # Do it

  identify_alarms();              # Count devices with alarms

  write_alarm_log();              # Record issues

  print_report();                 # Print report

  notify_staff();                 # Mail report
}

##### End Main Program #################################################



########################################################################
# Query variables
########################################################################
sub do_the_work {

  # Debug trace
  trace_location('begin') if $debug;

  # Notify operator
  print_it('Querying targets...');
  unless ($dome) {
    sleep $short;
    return 1;
  }

  # Loop through the list of targets
  for my $target (@target) {
    my ($val, $vb);
    say "Processing $target" if $debug;

    # Hardware failure logging enabled
    $val = snmpGet( {host=>$target, oid=>'cefcMIBEnableStatusNotification.0'} );
    $val //= $QUERY;
    unless ($val eq 'true') {
      log_it("For $target, Hardware Failure logging is disabled");
      $alarm_count{$target}++;
      push @{$alarms{$target}}, 'logging';
    }
    given ($val) {
      when ('true')  { $logging{$target} = 'yes' }
      when ('false') { $logging{$target} = 'no'  }
    }

    # Crypto accelerator status
    $val = snmpGet( {host=>$target, oid=>'ccaGlobalNumNonOperAccelerators.0'} );
    $val //= $QUERY;
    unless ($val == 0) {
      log_it("For $target, $val crypto accelerator cards have failed");
      $alarm_count{$target}++;
      push @{$alarms{$target}}, 'card failed';
    }
    $broken_card{$target} = $val;

    # Memory pools functional
    $vb = snmpWalk( {host => $target, oid => 'ciscoMemoryPoolValid'} );
    POOL:
    for my $varbind (@$vb) {
      my ($iid, $val);
      $iid = $varbind->{iid};
      $val = $varbind->{val};
      unless ($val eq 'true') {
        log_it("For $target, ciscoMemoryPoolValue.$iid is $val");
        $alarm_count{$target}++;
        push @{$alarms{$target}}, 'mem pool';
        $memory_pool_valid{$target} = 'no';
        last POOL;
      }
    }
    if (@$vb == 0) {
      log_it("For $target, unable to walk ciscoMemoryPoolValid");
      $memory_pool_valid{$target} = $QUERY;
    }
    else {
      $memory_pool_valid{$target} //= 'yes';
    }

    # Declined sessions
    $val = snmpGet( {host      => $target, 
                     oid       => 'cufwConnGlobalNumResDeclined.0', 
                     translate => 0}
                  );
    $val //= $QUERY;
    unless ($val == 0) {
      log_it("For $target, Declined Sessions = $val");
      $alarm_count{$target}++;
      push @{$alarms{$target}}, 'declined sessions';
    }
    $declined_sess{$target} = $val;

    # Primary status (cfwHardwareStatusValue.primaryUnit)
    $val = snmpGet( {host => $target, 
                     oid => '.1.3.6.1.4.1.9.9.147.1.2.1.1.1.3.6'}
                  );
    $val //= $QUERY;
    unless ($val eq 'active' or $val eq 'standby') {
      log_it("For $target, Primary Unit is $val");
      $alarm_count{$target}++;
      push @{$alarms{$target}}, 'HA status';
    }
    $primary_status{$target} = $val;

    # Secondary status (cfwHardwareStatusValue.secondaryUnit)
    $val = snmpGet( {host => $target, 
                     oid => '.1.3.6.1.4.1.9.9.147.1.2.1.1.1.3.7'}
                  );
    $val //= $QUERY;
    unless ($val eq 'active' or $val eq 'standby') {
      log_it("For $target, Secondary Unity is $val");
      $alarm_count{$target}++;
      push @{$alarms{$target}}, 'HA status';
    }
    $secondary_status{$target} = $val;

    # Compare primary and secondary status
    if ($primary_status{$target} eq $secondary_status{$target}) {
      log_it("Problem:  both sides of $target believe they are $primary_status{$target}");
      $alarm_count{$target}++;
      push @{$alarms{$target}}, 'HA status';
    }

    # netInterface (cfwHardwareStatusValue.netInterface)
    $val = snmpGet( {host => $target,
                     oid => '.1.3.6.1.4.1.9.9.147.1.2.1.1.1.3.4'}
                  );
    $val //= $QUERY;
    unless ($val eq 'up') {
      log_it("For $target, Net Interface is $val");
      $alarm_count{$target}++;
      push @{$alarms{$target}}, 'NIC';
    }
    $net_interface{$target} = $val;

    # Look at buffers
    $buffer_free{$target} = find_worst_buffer($target);
    if ($buffer_free{$target} eq 'low') {
      log_it("Buffers in $target are low");
      $alarm_count{$target}++;
      push @{$alarms{$target}}, 'buffers';
    }

    # Entertain operator
    print $BANG if $mode eq 'interactive';
  }

  # Make things look pretty
  say "\n" if $mode eq 'interactive';

  # Debug trace
  trace_location('end') if $debug;

  return 1;  
}




########################################################################
# Poke through cfwBufferStatValue, find the most heavily utilized buffer, 
# use $buffer_thresh_low and $buffer_thresh_high to calculate where we 
# stand, buffer-wise, in terms of utilization
########################################################################
sub find_worst_buffer {
  my $host = shift;
  my $low;
  my $max;
  my $util;
  my $val;
  my $worst = 'high';

  # Debug trace
  trace_location('begin') if $debug;

  # Analyze cfwBufferStatValue.0
#  $max = snmpGet( {host => $host, oid => 'cfwBufferStatValue.0.maximum'} );
#  $low = snmpGet( {host => $host, oid => 'cfwBufferStatValue.0.low'}     );
  $max = snmpGet( {host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.0.3'} );
  $low = snmpGet( {host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.0.5'} );
  if (defined $max and defined $low) {
    given ($low/$max) {
      when ($_ <= $buffer_thresh_low)  { $util = 'low'   }
      when ($_ <= $buffer_thresh_high) { $util = 'mid'   }
      default                          { $util = 'high'  }
    }
    given ($util) {
      when ('low') {
        log_it("For $host, worst-case cfwBufferStatValue.0 is $low");
        $alarm_count{$host}++;
        $worst = 'low';
      }
      when ('mid') { $worst = 'mid'  unless $worst eq 'low' }
    }
  }
  else {
    log_it("For $host, unable to retrieve cfwBufferStatValue.0");
  }

  # Analyze cfwBufferStatValue.4
#  $max = snmpGet( {host => $host, oid => 'cfwBufferStatValue.4.maximum'} );
#  $low = snmpGet( {host => $host, oid => 'cfwBufferStatValue.4.low'}     );
  $max = snmpGet( {host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.4.3'} );
  $low = snmpGet( {host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.4.5'} );
  if (defined $max and defined $low) {
    given ($low/$max) {
      when ($_ <= $buffer_thresh_low)  { $util = 'low'   }
      when ($_ <= $buffer_thresh_high) { $util = 'mid'   }
      default                          { $util = 'high'  }
    }
    given ($util) {
      when ('low') {
        log_it("For $host, worst-case cfwBufferStatValue.4 is $low");
        $alarm_count{$host}++;
        $worst = 'low';
      }
      when ('mid') { $worst = 'mid'  unless $worst eq 'low' }
    }
  }
  else {
    log_it("For $host, unable to retrieve cfwBufferStatValue.4");
  }

  # Analyze cfwBufferStatValue.80
#  $max = snmpGet( {host => $host, oid => 'cfwBufferStatValue.80.maximum'} );
#  $low = snmpGet( {host => $host, oid => 'cfwBufferStatValue.80.low'}     );
  $max = snmpGet( {host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.80.3'} );
  $low = snmpGet( {host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.80.5'} );
  if (defined $max and defined $low) {
    given ($low/$max) {
      when ($_ <= $buffer_thresh_low)  { $util = 'low'   }
      when ($_ <= $buffer_thresh_high) { $util = 'mid'   }
      default                          { $util = 'high'  }
    }
    given ($util) {
      when ('low') {
        log_it("For $host, worst-case cfwBufferStatValue.80 is $low");
        $alarm_count{$host}++;
        $worst = 'low';
      }
      when ('mid') { $worst = 'mid'  unless $worst eq 'low' }
    }
  }
  else {
    log_it("For $host, unable to retrieve cfwBufferStatValue.80");
  }

  # Analyze cfwBufferStatValue.256
#  $max = snmpGet( {host => $host, oid => 'cfwBufferStatValue.256.maximum'} );
#  $low = snmpGet( {host => $host, oid => 'cfwBufferStatValue.256.low'}     );
  $max = snmpGet({host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.256.3'});
  $low = snmpGet({host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.256.5'});
  if (defined $max and defined $low) {
    given ($low/$max) {
      when ($_ <= $buffer_thresh_low)  { $util = 'low'   }
      when ($_ <= $buffer_thresh_high) { $util = 'mid'   }
      default                          { $util = 'high'  }
    }
    given ($util) {
      when ('low') {
        log_it("For $host, worst-case cfwBufferStatValue.256 is $low");
        $alarm_count{$host}++;
        $worst = 'low';
      }
      when ('mid') { $worst = 'mid'  unless $worst eq 'low' }
    }
  }
  else {
    log_it("For $host, unable to retrieve cfwBufferStatValue.256");
  }

  # Analyze cfwBufferStatValue.1550
#  $max = snmpGet( {host => $host, oid => 'cfwBufferStatValue.1550.maximum'} );
#  $low = snmpGet( {host => $host, oid => 'cfwBufferStatValue.1550.low'}     );
  $max = snmpGet({host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.1550.3'});
  $low = snmpGet({host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.1550.5'});
  if (defined $max and defined $low) {
    given ($low/$max) {
      when ($_ <= $buffer_thresh_low)  { $util = 'low'   }
      when ($_ <= $buffer_thresh_high) { $util = 'mid'   }
      default                          { $util = 'high'  }
    }
    given ($util) {
      when ('low') {
        log_it("For $host, worst-case cfwBufferStatValue.1550 is $low");
        $alarm_count{$host}++;
        $worst = 'low';
      }
      when ('mid') { $worst = 'mid'  unless $worst eq 'low' }
    }
  }
  else {
    log_it("For $host, unable to retrieve cfwBufferStatValue.1550");
  }

  # Analyze cfwBufferStatValue.2048
#  $max = snmpGet( {host => $host, oid => 'cfwBufferStatValue.2048.maximum'} );
#  $low = snmpGet( {host => $host, oid => 'cfwBufferStatValue.2048.low'}     );
  $max = snmpGet({host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.2048.3'});
  $low = snmpGet({host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.2048.5'});
  if (defined $max and defined $low) {
    given ($low/$max) {
      when ($_ <= $buffer_thresh_low)  { $util = 'low'   }
      when ($_ <= $buffer_thresh_high) { $util = 'mid'   }
      default                          { $util = 'high'  }
    }
    given ($util) {
      when ('low') {
        log_it("For $host, worst-case cfwBufferStatValue.2048 is $low");
        $alarm_count{$host}++;
        $worst = 'low';
      }
      when ('mid') { $worst = 'mid'  unless $worst eq 'low' }
    }
  }
  else {
    log_it("For $host, unable to retrieve cfwBufferStatValue.2048");
  }

  # Analyze cfwBufferStatValue.2560
#  $max = snmpGet( {host => $host, oid => 'cfwBufferStatValue.2560.maximum'} );
#  $low = snmpGet( {host => $host, oid => 'cfwBufferStatValue.2560.low'}     );
  $max = snmpGet({host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.2560.3'});
  $low = snmpGet({host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.2560.3'});
  if (defined $max and defined $low) {
    given ($low/$max) {
      when ($_ <= $buffer_thresh_low)  { $util = 'low'   }
      when ($_ <= $buffer_thresh_high) { $util = 'mid'   }
      default                          { $util = 'high'  }
    }
    given ($util) {
      when ('low') {
        log_it("For $host, worst-case cfwBufferStatValue.2560 is $low");
        $alarm_count{$host}++;
        $worst = 'low';
      }
      when ('mid') { $worst = 'mid'  unless $worst eq 'low' }
    }
  }
  else {
    log_it("For $host, unable to retrieve cfwBufferStatValue.2560");
  }

  # Analyze cfwBufferStatValue.4096
#  $max = snmpGet( {host => $host, oid => 'cfwBufferStatValue.4096.maximum'} );
#  $low = snmpGet( {host => $host, oid => 'cfwBufferStatValue.4096.low'}     );
  $max = snmpGet({host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.4096.3'});
  $low = snmpGet({host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.4096.5'});
  if (defined $max and defined $low) {
    given ($low/$max) {
      when ($_ <= $buffer_thresh_low)  { $util = 'low'   }
      when ($_ <= $buffer_thresh_high) { $util = 'mid'   }
      default                          { $util = 'high'  }
    }
    given ($util) {
      when ('low') {
        log_it("For $host, worst-case cfwBufferStatValue.4096 is $low");
        $alarm_count{$host}++;
        $worst = 'low';
      }
      when ('mid') { $worst = 'mid'  unless $worst eq 'low' }
    }
  }
  else {
    log_it("For $host, unable to retrieve cfwBufferStatValue.4096");
  }

  # Analyze cfwBufferStatValue.8192
#  $max = snmpGet( {host => $host, oid => 'cfwBufferStatValue.8192.maximum'} );
#  $low = snmpGet( {host => $host, oid => 'cfwBufferStatValue.8192.low'}     );
  $max = snmpGet({host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.8192.3'});
  $low = snmpGet({host=>$host, oid=>'.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.8192.5'});
  if (defined $max and defined $low) {
    given ($low/$max) {
      when ($_ <= $buffer_thresh_low)  { $util = 'low'   }
      when ($_ <= $buffer_thresh_high) { $util = 'mid'   }
      default                          { $util = 'high'  }
    }
    given ($util) {
      when ('low') {
        log_it("For $host, worst-case cfwBufferStatValue.8192 is $low");
        $alarm_count{$host}++;
        $worst = 'low';
      }
      when ('mid') { $worst = 'mid'  unless $worst eq 'low' }
    }
  }
  else {
    log_it("For $host, unable to retrieve cfwBufferStatValue.8192");
  }

  # Analyze cfwBufferStatValue.16384
#  $max = snmpGet( {host => $host, oid => 'cfwBufferStatValue.16384.maximum'} );
#  $low = snmpGet( {host => $host, oid => 'cfwBufferStatValue.16384.low'}     );
  $max = snmpGet( {host => $host, oid => '.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.16384.3'} );
  $low = snmpGet( {host => $host, oid => '.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.16384.5'} );
  if (defined $max and defined $low) {
    given ($low/$max) {
      when ($_ <= $buffer_thresh_low)  { $util = 'low'   }
      when ($_ <= $buffer_thresh_high) { $util = 'mid'   }
      default                          { $util = 'high'  }
    }
    given ($util) {
      when ('low') {
        log_it("For $host, worst-case cfwBufferStatValue.16384 is $low");
        $alarm_count{$host}++;
        $worst = 'low';
      }
      when ('mid') { $worst = 'mid'  unless $worst eq 'low' }
    }
  }
  else {
    log_it("For $host, unable to retrieve cfwBufferStatValue.16384");
  }

  # Analyze cfwBufferStatValue.65536
#  $max = snmpGet( {host => $host, oid => 'cfwBufferStatValue.65536.maximum'} );
#  $low = snmpGet( {host => $host, oid => 'cfwBufferStatValue.65536.low'}     );
  $max = snmpGet( {host => $host, oid => '.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.65536.3'} );
  $low = snmpGet( {host => $host, oid => '.1.3.6.1.4.1.9.9.147.1.2.2.1.1.4.65536.3'} );
  if (defined $max and defined $low) {
    given ($low/$max) {
      when ($_ <= $buffer_thresh_low)  { $util = 'low'   }
      when ($_ <= $buffer_thresh_high) { $util = 'mid'   }
      default                          { $util = 'high'  }
    }
    given ($util) {
      when ('low') {
        log_it("For $host, worst-case cfwBufferStatValue.65536 is $low");
        $alarm_count{$host}++;
        $worst = 'low';
      }
      when ('mid') { $worst = 'mid'  unless $worst eq 'low' }
    }
  }
  else {
    log_it("For $host, unable to retrieve cfwBufferStatValue.65536");
  }

  # Debug trace
  trace_location('end') if $debug;

  return $worst;
}




########################################################################
# Tell the operator what I discovered
########################################################################
sub print_report {
  my $handle;
  my $total = @target;
  my $now = get_now();

  # If we are running in test mode, skip this routine
  unless ($dome) {
    print_it("Running in test mode, cannot print a meaningful report\n");
    return 1;
  }

  # Debug trace
  trace_location('begin') if $debug;

  # Direct output to screen or to file
  if ($mode eq 'interactive') {
    $handle = *STDOUT;
  }
  else {
    open $handle, '>', $report_file or die "Cannot open $report_file: $!\n";
  }


print {$handle} <<EOF;
# Title:           $report_subject
#
# Institution:     $institution
#
# Date of Report:  $now
#
# Description:     This report lists the status of hardware components
#
#                  Look through syslog for the details of alarms:
#                       grep $program_name /var/log/syslog
#
# Active:          $total
#
# Alarm Count:     $shit_happens
#
# In Alarm:        @alarm_list
# 
# Questions:       If you have questions or comments regarding this
#                  report, please mail them to $report_queries.
#
# Primary          Status of primary unit (active/standby/error)
# Secondary        Status of secondary unit
# Declined         # of sessions declined due to resource constraints
# Memory           Valid memory pools (yes/no)
# Cards            # of broken crypto accelerator cards
# Net              Status of network interface (up/down/error)
# Buffer           Free buffers (low/mid/high -- want mid or high)
# Logging          Logging of hardware issues (yes/no)
#
target           Primary  Secondary Declined Memory Cards Net Buffer Logging
---------------  -------- --------- -------- ------ ----- --- ------ -------
EOF
  for my $target (@target) {
    printf {$handle} 
      "%-15s  %-8s %-9s %-8s %-6s %-5s %-3s %-6s %-7s\n",
      $target, $primary_status{$target}, $secondary_status{$target}, 
      $declined_sess{$target}, $memory_pool_valid{$target}, 
      $broken_card{$target}, $net_interface{$target}, $buffer_free{$target}, 
      $logging{$target};
  }

  # Add silent devices to the report
  for my $silent (sort @silent) {
    printf {$handle} "%-15s  Not answering pings\n", $silent;
  }

  # Add unresponsive devices to the report
  for my $unresponsive (sort @unresponsive) {
    printf {$handle} "%-15s  Not answering SNMP GETs\n", $unresponsive;
  }

  # Add insane devices to the report
  for my $insane (sort @insane) {
    printf {$handle} "%-15s  Insane\n", $insane;
  }

  # Clean up
  unless ($handle =~ /STDOUT/) {
    close $handle or warn "Cannot close $report_file: $!\n";
  }

  # Debug trace
  trace_location('end') if $debug;

  return 1;
}




########################################################################
# Sanity check
########################################################################
sub sanity_check {
  my @remove;

  # Debug trace
  trace_location('begin') if $debug;

  # Notify operator
  print_it('Sanity check...');

  # Loop through targets, looking for problems
  TARGET:
  for my $target (@target) {
    my (%arg, $num);

    # Identify presence of CISCO-CRYPTO-ACCELERATOR-MIB support
    %arg = ( host => $target, oid => 'ccaSupportsHwCrypto.0' );
    unless (snmpGet(\%arg)) {
      say "\n$target does not support CISCO-CRYPTO-ACCELERATOR-MIB, ignoring" if $debug;
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      next TARGET;
    }

    # Identify presence of CISCO-UNIFIED-FIREWALL-MIB support
    %arg = ( host => $target, oid => 'cufwConnGlobalNumResDeclined.0' );
    $num = snmpGet(\%arg);
    unless (defined $num) {
      say "\n$target does not support CISCO-UNIFIED-FIREWALL-MIB, ignoring" if $debug;
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      next TARGET;
    }

    # Identify presence of CISCO-FIREWALL-MIB support
    %arg = ( host => $target, oid => 'cfwHardwareStatusValue.primaryUnit' );
    my $state = snmpGet(\%arg);
    unless (defined $num) {
      say "\n$target does not support CISCO-FIREWALL-MIB, ignoring" if $debug;
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      next TARGET;
    }
  }

  # Remove entries which failed checks
  prune_basic(@remove);
  @insane = @remove;

  # Make things look pretty
  say "\n" if $mode eq 'interactive';

  # Debug trace
  trace_location('end') if $debug;

  return 1;
}




########################################################################
# Output help
########################################################################
sub HELP_MESSAGE {
  print <<EOF;

  See $netops_global_config (and $netops_alarm_config if this is an
  xxxx-alarm script) for configuration options

  $usage

    -s asks the question:  are you serious?  If you answer 'no', then 
       the script will run in demo mode, making no changes

    -d specifies debug level (typically 0-9)

    -r determines whether or not the script writes its report to disk

    -a instructs the script to filter the hosts table using its internally
       defined suffixes variable to produce a list of targets

    -e requires a regex as its argument, with which the script filters
       the hosts table to produce a list of targets.  Enclose the regex in 
       quotes to protect special characters from the shell

    -f points the script to a file containing the list of targets

    target1 target2 target3 ... is the command-line list of targets you
      have specified, in lieu of the -a, -e, or -f options

EOF

  return 1;
}




########################################################################
# Output version
########################################################################
sub VERSION_MESSAGE {
  say "$program_name v$version";
  return 1;
}