#!/opt/vdops/bin/perl

# This script identifies ports on Cisco switches which have dropped packets 
# from either input or output queues

# V      Who       When        What
# ---------------------------------------------------------------------------
# 2.2.0  skendric  2010-04-21  Upgrade to perl 5.10.1
# 2.1.0  skendric  2009-02-08  Ignore interfaces with low traffic volume and
#                              interfaces on internally defined skip list
# 2.0.1  skendric  2009-02-06  Fiddle with help message
# 2.0.0  skendric  2007-09-18  Snapshot of four error counters vs ifName
# 1.9.2  skendric  2007-09-18  Smarter about correlating interface stats
# 1.9.1  skendric  2007-09-18  Add thresholds and ifIn/OutErrors
# 1.0.0  skendric  2007-09-17  First Version
#

# Author:  Stuart Kendrick, sbk {put at sign here} skendric {put dot here} com
#
# Source:  http://www.skendric.com/device
#
# This software is available under the GNU GENERAL PUBLIC LICENSE, see
# http://www.fsf.org/licenses/gpl.html
#

#
# This script takes the following approach:
#     -Parses the hosts table for a list of targets (or accepts a command-
#      line list)
#     -Identifies the ports on switches where the percentage of ifInErrors, 
#      ifOutErrors, ifInDiscards, or ifOutDiscards exceeds specified 
#      thresholds
#     -Produces a report lists these switches/ports
#
#
# Requirements:
#     -The following MIB modules stashed in /opt/vdops/share/snmp/mibs,
#      or wherever it is that you store MIB modules:
#      CISCO-PRODUCTS-MIB.my
#
#     -PERL modules:  the FHCRC::Netops collection
#
#
# Assumptions:
#
#
# Tested on:
#       -perl-5.10.1
#       -net-snmp-5.5
#
#
# Instructions:
#    -Customize the script for your site:  find the 'user-configurable
#     variables' section and modify as appropriate
#    -Try it out
#
#
#
# Caveats:
#
#
# Known Bugs:
#
#
# To do:
#     -In the report, specify which modules contain afflicted ports
#     -Add support for SNMPv3
#

# Begin script

# Load modules
use strict;
use warnings;
use feature 'say';
use feature 'switch';
use Carp qw(carp cluck croak confess);
use Data::Dumper;
use English qw( -no_match_vars );
use Getopt::Std;
use List::MoreUtils qw(uniq);
use Regexp::Common;
use FHCRC::Netops::CiscoTools 1.3.1;
use FHCRC::Netops::HostTools 1.0.3;
use FHCRC::Netops::NetopsTools 2.0.7;
use FHCRC::Netops::NetopsData 1.3.0;
use FHCRC::Netops::PingTools 1.1.5;
use FHCRC::Netops::SNMPTools 1.3.9;
use FHCRC::Netops::Utilities 1.3.9;

# Declare global variables
my $error_thresh;              # User-specified threshold of errors, expressed 
                               # as a percentage of (error packets)/(total 
                               # packets)
my $discard_thresh;            # User-specified threshold of discards, 
                               # expressed as a percentage of (error packets)/
                               # (total packets)
my %if_problem;                # Hash of references to hashes encapsulating 
                               # the fields printed in the report, keyed by 
                               # target
my %if_name;                   # Hash of arrays references to arrays of ifName 
                               # walks, keyed by target
my %if_oper_status;            # Hash of references to arrays of ifOperStatus
                               # walks, keyed by target
my %if_in_discards;            # Hash of references to arrays of ifInDiscards
                               # walks, keyed by target
my %if_out_discards;           # Hash of references to arrays of ifOutDiscards
                               # walks, keyed by target
my %if_in_errors;              # Hash of references to arrays of ifInErrors
                               # walks, keyed by target
my %if_out_errors;             # Hash of references to arrays of ifOutErrors
                               # walks, keyed by target
my %if_hc_in_ucast_pkts;       # Hash of references to arrays of 
                               # ifHCInUcastPkts walks, keyed by target
my %if_hc_out_ucast_pkts;      # Hash of references to arrays of 
                               # ifHCOutUcastPkts walks, keyed by target
my %if_hc_in_broadcast_pkts;   # Hash of references to arrays of 
                               # ifHCInBroadcastPkts walks, keyed by target
my %if_hc_out_broadcast_pkts;  # Hash of references to arrays of 
                               # ifHCOutBroadcastPkts walks, keyed by target
my %if_hc_in_multicast_pkts;   # Hash of references to arrays of 
                               # ifHCInMulticastPkts walks, keyed by target
my %if_hc_out_multicast_pkts;  # Hash of references to arrays of 
                               # ifHCOutMulticastPkts walks, keyed by target
my $log_percent;               # Boolean telling us whether or not to log 
                               # error/discard percentages per interface
my %skip_interface;            # Hash of switches keyed by switchName-interface 
                               # These are interfaces with known issues which 
                               # we want to ignore, i.e. which we know about 
                               # and which we do not want triggering e-mail
my $volume_thresh;             # User-specified number of frames below which 
                               # we automatically ignore the interface.  The 
                               # thinking here is that if the interface has 
                               # seen some small number of frames, say '10', 
                               # and two of those were errors, then normally 
                               # this interface would show up on the report, 
                               # as having a 20% error rate.  But really, 
                               # unless we've seen enough traffic, the error 
                               # (or discard) percentage doesn't point us 
                               # toward a problem.


# Define global variables
$debug = 0;
            # 10 = Logging
            #  9 = Database SELECT operations
            #  8 = Per IP/MAC/Port processing
            #  7 = Database INSERT/UPDATE/DELETE
            #  6 = Dump SNMP var
            #  5 = Dump snmp_packets
            #  4 = Grody:    print big var
            #  3 = Verbose:  print mid var
            #  2 = Simple:   print small var
            #  1 = Basic:    subroutine trace
            #  0 = Disable debugging
$program_name = 'find-if-problems';
$usage = 'Usage:  find-if-problems -s {yes|no} [-d {integer}] [-r] [-a | -e {expr} | -f {filename} | target1 target2 target3 ...]'; 
$version = '2.2.0';

# Binaries
$grab_hosts = '/bin/cat /etc/hosts';

# Logging
$log_percent = 0;

# Pause parameters
$long  = 30;
$mid   = 10;
$short = 2;

# Ping Stuff
$ping_count   = 3;
$ping_timeout = 1;

# Report stuff
$institution       = 'Widgets International';
$owner             = 'Stuart Kendrick';
$owner_backup      = 'Stuart Kendrick';
$report_file       = '/home/netops/rpts/find-if-problems.txt';
$report_queries    = 'bsmith@widgets.com';
$report_recipients = 'skendric@fhcrc.org';
$report_subject    = 'Problem Interface Report';

# Thresholds
$error_thresh   = .1;
$discard_thresh = 1;
$volume_thresh  = 10000;

# SNMP Stuff
# Optimize performance by sorting your community strings and SNMP version
# list, most frequently used to the left, least frequently used to the right
@mib_dir           = qw(/opt/vdops/share/snmp/mibs);
@mib_file          = qw/ALL/;
@snmp_read_list    = qw/public/;
@snmp_version_list = qw/2/;
$snmp_port         = 161;
$snmp_retries      = 3;
$snmp_timeout      = 3000000;

# Syslog stuff
$syslog_facility  = 'local5';
$syslog_host      = 'localhost';
$syslog_port      = 514;        
$syslog_priority  = 'info';
$syslog_socket    = 'unix';    # Other possibilites include 'udp' and
                               # 'stream'; depending on the flavor of Unix,
                               # I've employed each of these

# Target details
%skip_interface = (
                    'dfsr-b-esx Gi3/19' => 'iota',
                  );
@skip_name = qw/swamp/;
@suffixes  = qw/-esx -rtr/;


# Grab arguments
getopts('ab:d:e:f:rs:t:v:', \%option);
@target = @ARGV;
$error_thresh   = $option{b} if defined $option{b};
$discard_thresh = $option{t} if defined $option{t};
$volume_thresh  = $option{v} if defined $option{v};
unless ($RE{num}{real}->matches($error_thresh)) {
  die "-b {num} must be a number\n" ;
}
unless ($RE{num}{real}->matches($discard_thresh)) {
  die "-t {num} must be a number\n" ;
}
unless ($RE{num}{real}->matches($volume_thresh)) {
  die "-v {num} must be a number\n" ;
}

# Set mode
if      ($option{r})            { $mode = 'report'      }
elsif   (-t STDIN)              { $mode = 'interactive' }
else                            { $mode = 'batch'       }


### Begin Main Program ###############################################

{

  check_args();                   # Check arguments

  compile_mibs();                 # Compile MIB files

  build_target();                 # Populate @target

  target_check();                 # Look for errors in @target

  basic_info();                   # Gather information

  info_before();                  # Gather more information

  #sanity_check();                 # Sanity check

  do_the_work();                  # Do the work

  identify_alarms();              # Count devices with alarms

  print_report();                 # Print report

  notify_staff();                 # Mail report
}

##### End Main Program ###############################################



########################################################################
# Do the work
########################################################################
sub do_the_work {

  # Debug trace
  trace_location('begin') if $debug;

  # Notify operator
  print_it('Calculating interface discard percentages...');

  # Loop through targets
  TARGET:
  for my $target (@target) {
    say "Processing $target" if $debug > 1;

    # Walk through interfaces
    INTERFACE:
    for my $varbind (@{$if_oper_status{$target}}) {
      my ($if_index, $if_name, $if_oper_status);
      my ($if_in_errors, $if_out_errors, $if_in_discards, $if_out_discards);
      my ($if_in_pkts, $if_out_pkts);
      my ($if_in_error_pct, $if_out_error_pct);
      my ($if_in_discard_pct, $if_out_discard_pct);

      # Extract ifOperStatus
      $if_index       = $varbind->{iid};
      $if_oper_status = $varbind->{val};

      # Find ifName
      IFNAME:
      for my $varbind (@{$if_name{$target}}) {
        if ($if_index == $varbind->{iid}) {
          $if_name = $varbind->{val};
          last IFNAME;
        }
      }
      $if_name = $QUERY unless defined $if_name;

      # Debug info
      say "Processing $if_name" if $debug == 8;
        
      # Skip interfaces which are down
      unless ($if_oper_status eq 'up') {
        print_it("$if_name is down or testing, skipping") if $debug == 8;
        next INTERFACE;
      }

      # Extract ifInError count
      INERROR:
      for my $varbind (@{$if_in_errors{$target}}) {
        if ($if_index == $varbind->{iid}) {
          $if_in_errors = $varbind->{val};
          last INERROR;
        }
      }
      $if_in_errors = 0 unless defined $if_in_errors;

      # Extract ifOutErrors counts
      OUTERROR:
      for my $varbind (@{$if_out_errors{$target}}) {
        if ($if_index == $varbind->{iid}) {
          $if_out_errors = $varbind->{val};
          last OUTERROR;
        }
      }
      $if_out_errors = 0 unless defined $if_out_errors;

      # Extract ifInDiscards counts
      INDISCARD:
      for my $varbind (@{$if_in_discards{$target}}) {
        if ($if_index == $varbind->{iid}) {
          $if_in_discards = $varbind->{val};
          last INDISCARD;
        }
      }
      $if_in_discards = 0 unless defined $if_in_discards;

      # Extract ifOutDiscards counts
      OUTDISCARD:
      for my $varbind (@{$if_out_discards{$target}}) {
        if ($if_index == $varbind->{iid}) {
          $if_out_discards = $varbind->{val};
          last OUTDISCARD;
        }
      }
      $if_out_discards = 0 unless defined $if_out_discards;

      # Unless at least one of these is non-zero, skip to the next interface
      unless (   $if_in_errors   > 0 or $if_out_errors   > 0 
              or $if_in_discards > 0 or $if_out_discards > 0 ) {
        print_it("$if_name reports no problems, skipping") if $debug == 8;
        next INTERFACE;
      }

      # Extract ifHCInUcastPkts
      my $in_unicast;
      INUCAST:
      for my $varbind (@{$if_hc_in_ucast_pkts{$target}}) {
        if ($if_index == $varbind->{iid}) {
          $in_unicast = $varbind->{val};
          last INUCAST;
        }
      }
      $in_unicast = 0 unless defined $in_unicast;

      # Extract ifHCOutUcastPkts
      my $out_unicast;
      OUTUCAST:
      for my $varbind (@{$if_hc_out_ucast_pkts{$target}}) {
        if ($if_index == $varbind->{iid}) {
          $out_unicast = $varbind->{val};
          last OUTUCAST;
        }
      }
      $out_unicast = 0 unless defined $out_unicast;

      # Extract ifHCInBroadcastPkts
      my $in_broadcast;
      INBROADCAST:
      for my $varbind (@{$if_hc_in_broadcast_pkts{$target}}) {
        if ($if_index == $varbind->{iid}) {
          $in_broadcast = $varbind->{val};
          last INBROADCAST;
        }
      }
      $in_broadcast = 0 unless defined $in_broadcast;

      # Extract ifHCOutBroadcastPkts
      my $out_broadcast;
      OUTBROADCAST:
      for my $varbind (@{$if_hc_out_broadcast_pkts{$target}}) {
        if ($if_index == $varbind->{iid}) {
          $out_broadcast = $varbind->{val};
          last OUTBROADCAST;
        }
      }
      $out_broadcast = 0 unless defined $out_broadcast;

      # Extract ifHCInMulticastPkts
      my $in_multicast;
      INMULTICAST:
      for my $varbind (@{$if_hc_in_multicast_pkts{$target}}) {
        if ($if_index == $varbind->{iid}) {
          $in_multicast = $varbind->{val};
          last INMULTICAST;
        }
      }
      $in_multicast = 0 unless defined $in_multicast;

      # Extract ifHCOutMulticastPkts
      my $out_multicast;
      OUTMULTICAST:
      for my $varbind (@{$if_hc_out_multicast_pkts{$target}}) {
        if ($if_index == $varbind->{iid}) {
          $out_multicast = $varbind->{val};
          last OUTMULTICAST;
        }
      }
      $out_multicast = 0 unless defined $out_multicast;


      # Calculate input packets
      $if_in_pkts = $in_unicast + $in_broadcast + $in_multicast;
      if ($if_in_pkts == 0) {
        if ($debug == 8) {
          print_it("$target:$if_name sees no input packets, skipping");
        }
        next INTERFACE;
      } 
 
      # Calculate output packets
      $if_out_pkts = $out_unicast + $out_broadcast + $out_multicast;
      if ($if_out_pkts == 0) {
        if ($debug == 8) {
          print_it("$target:$if_name sees no output packets, skipping");
        }
        next INTERFACE;
      } 

      # Calculate percentages
      $if_in_error_pct    = sprintf "%2.3f", ($if_in_errors / $if_in_pkts);
      $if_out_error_pct   = sprintf "%2.3f", ($if_out_errors / $if_out_pkts); 
      $if_in_discard_pct  = sprintf "%2.3f", ($if_in_discards / $if_in_pkts);
      $if_out_discard_pct = sprintf "%2.3f", ($if_out_discards / $if_out_pkts);

      # Log percentages if asked
      if ($log_percent == 1) {
        log_it("For $target:$if_name, if_in_error_pct = $if_in_error_pct, if_out_error_pct = $if_out_error_pct, if_in_discard_pct = $if_in_discard_pct, if_out_discard_pct = $if_out_discard_pct"); 
      }

      # Skip to next interface if this interface belongs to a skip list
      next INTERFACE if defined $skip_interface{"$target $if_name"};

      # Compare to thresholds
      my $problem = 0;
      if ($if_in_pkts > $volume_thresh) {
        $problem = 1 if $if_in_error_pct    >= $error_thresh;
        $problem = 1 if $if_in_discard_pct  >= $discard_thresh;
      }
      if ($if_out_pkts > $volume_thresh) {
        $problem = 1 if $if_out_error_pct   >= $error_thresh;
        $problem = 1 if $if_out_discard_pct >= $discard_thresh;
      }

      # Skip ahead if error/discard counts don't reach thresholds
      next INTERFACE unless $problem == 1;

      # Build hash of percentages
      my %percentage = (
                         if_name            => $if_name,
                         if_in_error_pct    => $if_in_error_pct,
                         if_out_error_pct   => $if_out_error_pct,
                         if_in_discard_pct  => $if_in_discard_pct,
                         if_out_discard_pct => $if_out_discard_pct,
                       );

      # Record problem interface
      $alarm_count{$target}++;
      push @{$if_problem{$target}}, \%percentage;
      log_it("Found problems on $target:$if_name");

    }  # End 'Walk through interfaces'

    # Entertain operator
    print $BANG if $mode eq 'interactive';

  }  # End 'Loop through targets'

  # Make things look pretty
  say "\n" if $mode eq 'interactive';

  # Debug trace
  trace_location('end') if $debug;

  return 1;
}




########################################################################
# Gather more information
########################################################################
sub info_before { 
  my @remove;

  # Debug trace
  trace_location('begin') if $debug;

  # Notify operator
  print_it('Gathering interface attributes...');

  # Loop through targets, acquiring interface counters
  TARGET:
  for my $target (@target) {   
    my $ref;
    say "Processing $target" if $debug;

    # Acquire ifName
    say 'Walking ifName' if $debug > 3;
    $ref = snmpWalk( {host => $target, oid => $ifName_oid} );
    say Dumper($ref) if $debug == 8;
    unless (defined $ref and @$ref > 0) {
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      print_it("$target not responding to ifName walk, skipping");
      next TARGET;
    }
    $if_name{$target} = $ref;
    print $DASH if $mode eq 'interactive';
    sleep $short;
  
    # Acquire ifOperStatus
    $ref = snmpWalk( {host => $target, oid => 'ifOperStatus'} );
    say Dumper($ref) if $debug == 8;
    unless (defined $ref and @$ref > 0) {
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      print_it("$target not responding to ifOperStatus walk, skipping");
      next TARGET;
    }
    $if_oper_status{$target} = $ref;
    print $DASH if $mode eq 'interactive';
    sleep $short;

    # Acquire ifInDiscards
    $ref = snmpWalk( {host => $target, oid => 'ifInDiscards'} );
    say Dumper($ref) if $debug == 8;
    unless (defined $ref and @$ref > 0) {
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      print_it("$target not responding to ifInDiscards walk, skipping");
      next TARGET;
    }
    $if_in_discards{$target} = $ref;
    print $DASH if $mode eq 'interactive';
    sleep $short;
  
    # Acquire ifInErrors
    $ref = snmpWalk( {host => $target, oid => 'ifInErrors'} );
    say Dumper($ref) if $debug == 8;
    unless (defined $ref and @$ref > 0) {
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      print_it("$target not responding to ifInErrors walk, skipping");
      next TARGET;
    }
    $if_in_errors{$target} = $ref;
    print $DASH if $mode eq 'interactive';
    sleep $short;
  
    # Acquire ifOutDiscards
    $ref = snmpWalk( {host => $target, oid => 'ifOutDiscards'} );
    say Dumper($ref) if $debug == 8;
    unless (defined $ref and @$ref > 0) {
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      say "$target not responding to ifOutDiscards walk, skipping" if $debug;
      next TARGET;
    }
    $if_out_discards{$target} = $ref;
    print $DASH if $mode eq 'interactive';
    sleep $short;
  
    # Acquire ifOutErrors
    say 'Walking ifOutErrors' if $debug > 3;
    $ref = snmpWalk( {host => $target, oid => 'ifOutErrors'} );
    say Dumper($ref) if $debug == 8;
    unless (defined $ref and @$ref > 0) {
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      print_it("$target not responding to ifOutErrors walk, skipping");
      next TARGET;
    }
    $if_out_errors{$target} = $ref;
    print $DASH if $mode eq 'interactive';
    sleep $short;
  
    # Acquire ifHCInUcastPkts
    say 'Walking ifHCInUcastPkts' if $debug > 3;
    $ref = snmpWalk( {host => $target, oid => 'ifHCInUcastPkts'} );
    unless (defined $ref and @$ref > 0) {
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      say "$target not responding to ifHCInUcastPkts walk, skipping" if $debug;
      next TARGET;
    }
    $if_hc_in_ucast_pkts{$target} = $ref;
    print $DASH if $mode eq 'interactive';
    sleep $short;
  
    # Acquire ifHCInMulticastPkts
    $ref = snmpWalk( { host => $target, oid => 'ifHCInMulticastPkts'} );
    say Dumper($ref) if $debug == 8;
    unless (defined $ref and @$ref > 0) {
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      say "$target not responding to ifHCInMulticastPkts walk, skipping" if $debug;
      next TARGET;
    }
    $if_hc_in_multicast_pkts{$target} = $ref;
    print $DASH if $mode eq 'interactive';
    sleep $short;
  
    # Acquire ifHCInBroadcastPkts
    $ref = snmpWalk( {host => $target, oid => 'ifHCInBroadcastPkts'} );
    say Dumper($ref) if $debug == 8;
    unless (defined $ref and @$ref > 0) {
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      say "$target not responding to ifHCInBroadcastPkts walk, skipping" if $debug;
      next TARGET;
    }
    $if_hc_in_broadcast_pkts{$target} = $ref;
    print $DASH if $mode eq 'interactive';
    sleep $short;
  
    # Acquire ifHCOutUcastPkts
    $ref = snmpWalk( {host => $target, oid => 'ifHCOutUcastPkts'} );
    say Dumper($ref) if $debug == 8;
    unless (defined $ref and @$ref > 0) {
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      say "$target not responding to ifHCOutUcastPkts walk, skipping" if $debug;
      next TARGET;
    }
    $if_hc_out_ucast_pkts{$target} = $ref;
    print $DASH if $mode eq 'interactive';
    sleep $short;
  
    # Acquire ifHCOutMulticastPkts
    say 'Walking ifHCOutMulticastPkts' if $debug > 3;
    $ref = snmpWalk( {host => $target, oid => 'ifHCOutMulticastPkts'} );
    say Dumper($ref) if $debug == 8;
    unless (defined $ref and @$ref > 0) {
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      say "$target not responding to ifHCOutMulticastPkts walk, skipping" if $debug;
      next TARGET;
    }
    $if_hc_out_multicast_pkts{$target} = $ref;
    print $DASH if $mode eq 'interactive';
    sleep $short;
  
    # Acquire ifHCOutBroadcastPkts
    say 'Walking ifHCOutBroadcastPkts' if $debug > 3;
    $ref = snmpWalk( {host => $target, oid => 'ifHCOutMulticastPkts'} );
    say Dumper($ref) if $debug == 8;
    unless (defined $ref and @$ref > 0) {
      push @remove, $target;
      print $DOT if $mode eq 'interactive';
      say "$target not responding to ifHCOutBroadcastPkts walk, skipping" if $debug;
      next TARGET;
    }
    $if_hc_out_broadcast_pkts{$target} = $ref;
    print $DASH if $mode eq 'interactive';
  
    print $BANG if $mode eq 'interactive';
  }

  # Remove entries which didn't return all data
  if (@remove > 0) {
    my $silent = join $SPACE, @remove;
    print_it("\nThe following targets did not respond to a walk: $silent");
    prune_basic(@remove);
  }

  # Make things look pretty
  say "\n" if $mode eq 'interactive';

  # Debug trace
  trace_location('end') if $debug;

  return 1;
}




########################################################################
# Tell the operator what I discovered
########################################################################
sub print_report {
  my @fields;
  my $handle;
  my $total = @target;
  my $now = get_now();

  # Debug trace
  trace_location('begin') if $debug;

  # If we are running in test mode, skip this routine
  unless ($dome) {
    print_it("Running in test mode, cannot print a meaningful report\n");
    return 1;
  }

  # Count number of switches
  $shit_happens = keys %if_problem;

  # Direct output to screen or to file
  if ($mode eq 'interactive') {
    $handle = *STDOUT;
  }
  else {
    open $handle, '>', $report_file or die "Cannot open $report_file: $!\n";
  }

print {$handle} <<EOF;
# Title:           $report_subject
#
# Institution:     $institution
#
# Date of Report:  $now
#
# Description:     This report identifies interfaces which have experienced 
#                  input or output queue drops or physical layer errors
#                  exceeding $discard_thresh\% (discards) or $error_thresh\% (errors)
#
# Active:          $total
#
# Problems:        $shit_happens
#
# Questions:       If you have questions or comments regarding this
#                  report, please mail them to $report_queries.
#
#
# target              ifName      %InError  %OutError  %InDiscard  %OutDiscard
# ------------------  ----------  --------  ---------  ----------  -----------
EOF

  # Walk targets
  for my $target (sort keys %if_problem) {
    my ($first, $line);
    $first = 1;

    # Walk list of interfaces for this target
    for my $int (@{$if_problem{$target}}) {
      my ($if_name, $in_error, $out_error, $in_discard, $out_discard);
      $if_name = $int->{if_name};
      $in_error = $int->{if_in_error_pct};
      $out_error = $int->{if_out_error_pct};
      $in_discard = $int->{if_in_discard_pct};
      $out_discard = $int->{if_out_discard_pct};

      if ($first == 1) {
        printf {$handle} "  %-18s  %-10s  %2.3f     %2.3f      %2.3f       %2.3f\n", $target, $if_name, $in_error, $out_error, $in_discard, $out_discard;
        $first = 0;
      }
      else {
        printf {$handle} "                      %-10s  %2.3f     %2.3f      %2.3f       %2.3f\n", $if_name, $in_error, $out_error, $in_discard, $out_discard;
      }

    }
  
  }   
        
  unless ($handle =~ /STDOUT/) {
    close $handle or warn "Cannot close $report_file: $!\n";
  }

  # Debug trace
  trace_location('end') if $debug;

  return 1;
}



########################################################################
# Sanity check
########################################################################
sub sanity_check {
  my @remove;

  # Debug trace
  trace_location('begin') if $debug;

  # Notify operator
  print_it('Sanity check...');

  # Loop through targets, looking for problems
  TARGET:
  for my $target (@target) {   
    my ($int, $num_interfaces);
  
    # Each data array should be the same size, meaning, each array 
    # should contain the same number of elements, because each array 
    # tracks the same list of interfaces from the same device
    $num_interfaces = @{$if_name{$target}};
    print_it("$target contains $num_interfaces interfaces") if $debug > 2;

    my $oper_int = @{$if_oper_status{$target}};
    unless ($num_interfaces == $oper_int) {
      print_it("$target contains $oper_int oper_interfaces") if $debug > 2;
      push @remove, $target;
    }

    $int = @{$if_oper_status{$target}};
    unless ($num_interfaces == $int) {
      print_it("$target contains $int oper_interfaces") if $debug > 2;
      push @remove, $target;
    }

    $int = @{$if_in_discards{$target}};
    unless ($num_interfaces == $int) {
      print_it("$target contains $int in_discards") if $debug > 2;
      push @remove, $target;
    }

    $int = @{$if_out_discards{$target}};
    unless ($num_interfaces == $int) {
      print_it("$target contains $int out_discards") if $debug > 2;
      push @remove, $target;
    }

    $int = @{$if_in_errors{$target}};
    unless ($num_interfaces == $int) {
      print_it("$target contains $int in_errors") if $debug > 2;
      push @remove, $target;
    }

    $int = @{$if_out_errors{$target}};
    unless ($num_interfaces == $int) {
      print_it("$target contains $int out_errors") if $debug > 2;
      push @remove, $target;
    }

    $int = @{$if_hc_in_ucast_pkts{$target}};
    unless ($num_interfaces == $int) {
      print_it("$target contains $int in_ucast") if $debug > 2;
      push @remove, $target;
    }

    $int = @{$if_hc_out_ucast_pkts{$target}};
    unless ($num_interfaces == $int) {
      print_it("$target contains $int out_ucast") if $debug > 2;
      push @remove, $target;
    }

    $int = @{$if_hc_in_broadcast_pkts{$target}};
    unless ($num_interfaces == $int) {
      print_it("$target contains $int in_broadcast") if $debug > 2;
      push @remove, $target;
    }

    $int = @{$if_hc_out_broadcast_pkts{$target}};
    unless ($num_interfaces == $int) {
      print_it("$target contains $int out_broadcast") if $debug > 2;
      push @remove, $target;
    }

    $int = @{$if_hc_in_multicast_pkts{$target}};
    unless ($num_interfaces == $int) {
      print_it("$target contains $int in_multicast") if $debug > 2;
      push @remove, $target;
    }

    $int = @{$if_hc_out_multicast_pkts{$target}};
    unless ($num_interfaces == $int) {
      print_it("$target contains $int out_multicast") if $debug > 2;
      push @remove, $target;
    }

    print $BANG if $mode eq 'interactive';
  }

  # Sanitize @remove
  @remove = uniq @remove;

  # Notify operator
  if (@remove > 0) { 
    say('');
    say 'The following targets responded with inconsistent interface counts, skipping';
    say join $SPACE, @remove;
    say('');
  }

  # Remove entries which failed checks
  prune_basic(@remove);

  # Make things look pretty
  say "\n" if $mode eq 'interactive';

  # Debug trace
  trace_location('end') if $debug;

  return 1;
}




########################################################################
# Output help
########################################################################
sub HELP_MESSAGE {
  print <<EOF;

  $usage

    -s asks the question:  are you serious?  If you answer 'no', then 
       the script will run in demo mode, making no changes

    -b specifies the error ('bad') threshold, above which the report will 
       include the interface; the default .1 (i.e. if the percentage of bad 
       packets to total packets exceeds .1%, include the interface in the 
       report)

    -t specifies the discard ('toss') threshold, above which the report will 
       include the interface; the default is .1 (i.e. if the percentage of 
       tossed packets to total packets exceeds .1%, include the interface in 
       the report)
    
    -v specifies the volume threshold below which we will automatically ignore 
       this interface.  Useful for screening out interfaces which report a 
       high percentage of errors or discards based on a low number of total 
       frames.  Default is 10,000

    -d specifies debug level (typically 0-9)

    -r determines whether or not the script writes its report to disk

    -a instructs the script to filter the hosts table using its internally
       defined suffixes variable to produce a list of targets

    -e requires a regex as its argument, with which the script filters
       the hosts table to produce a list of targets.  Enclose the regex in 
       quotes to protect special characters from the shell

    -f points the script to a file containing the list of targets

    target1 target2 target3 ... is the command-line list of targets you
      have specified, in lieu of the -a, -e, or -f options

EOF

  return 1;
}




########################################################################
# Output version
########################################################################
sub VERSION_MESSAGE {
  say "$program_name v$version";
  return 1;
}