#!/opt/vdops/bin/perl

# This script queries PRI-related parameters from IF-MIB and DS1-MIB, 
# producing a report on the results and emitting e-mail if it 
# discovers a problem

# V      Who       When        What
# ---------------------------------------------------------------------------
# 1.2.0  skendric  2011-02-21  Upgrade to Netops 1.4.0
# 1.1.2  skendric  2010-12-30  Add @insane to report
# 1.1.1  skendric  2010-12-17  Futz with owner/owner_backup
# 1.1.0  skendric  2010-04-14  Upgrade to perl 5.10.1
# 1.0.7  skendric  2009-05-21  Support SNMP.pm
# 1.0.6  skendric  2009-03-20  Add @down_for_maintenance
# 1.0.5  skendric  2008-01-02  Ignore dsx1 errors
# 1.0.4  skendric  2008-05-21  Report fiddles
# 1.0.3  skendric  2008-05-16  Enhance logging of interface errors
# 1.0.2  skendric  2008-04-28  Log errors when we encounter them
# 1.0.1  skendric  2008-02-25  Additional debug output
# 1.0.0  skendric  2008-02-16  First Version
#
#
# Author:  Stuart Kendrick, sbk {put at sign here} skendric {put dot here} com
#
# Source:  http://www.skendric.com/device
#
# This software is available under the GNU GENERAL PUBLIC LICENSE, see
# http://www.fsf.org/licenses/gpl.html
#

#
# This script takes the following approach:
#     -Parses the hosts table for a list of targets (or accepts a command-
#      line list)
#     -Queries a bunch of CISCO specific variables
#     -Produces a report
#
#
# Requirements:
#      -The target(s) must be pingable
#
#      -The following MIB modules stashed in /opt/vdops/share/snmp/mibs,
#       or wherever it is that you store MIB modules:
#       CISCO-PRODUCTS-MIB.my
#
#     -PERL modules:  the WI::Netops collection
#
#
# Assumptions:
#
#
# Tested on:
#     -perl-5.12.2
#     -net-snmp-5.6
#
#
# Instructions:
#     -Customize the script for your site:  find the 'user-configurable
#      variables' section and modify as appropriate
#     -Try it out
#
#
# Caveats:
#
#
# Known Bugs:
#
#
# To do:
#     -Add support for SNMPv3
#

# Begin script

# Load modules
use strict;
use warnings;
use feature 'say';
use feature 'switch';
use Carp qw(carp cluck croak confess);
use Data::Dumper;
use English qw( -no_match_vars );
use Getopt::Std;
use List::MoreUtils qw(any);
use WI::Netops::CiscoTools 1.4.3;
use WI::Netops::HostTools 1.0.4;
use WI::Netops::IFTools 1.3.1;
use WI::Netops::NetopsTools 2.2.3;
use WI::Netops::NetopsData 1.4.0;
use WI::Netops::PingTools 1.1.7;
use WI::Netops::SNMPTools 1.5.3;
use WI::Netops::Utilities 1.4.4;

# Declare global variables.  All hashes keyed by target
my %if_info;         # Hash of hash refs, keyed by target name
#  e.g.
#  %if_info = (
#    target1 => {
#      1 => { %ifStuff },
#      2 => { %ifStuff },
#      3 => { %ifStuff },
#    }
#    target2 => {
#      15 => { %ifStuff },
#      21 => { %ifStuff },
#      33 => { %ifStuff },
#    }
#  );
#  
#  %ifStuff = (
#    'ifDescr'                 => 'T1 0/0/0',
#    'ifType'                  => 'ds1',
#    'ifAdminStatus            => 'up',
#    'ifOperStatus             => 'up',
#    'ifInErrors'              => '0',
#    'ifOutErrors'             => '0',
#    'dsx1LineStatus'          => 'NoAlarm',
#    'dsx1CurrentESs'          => 0,
#    'dsx1CurrentSESs'         => 0,
#    'dsx1CurrentSEFSs'        => 0,
#    'dsx1CurrentUASs'         => 0,
#    'dsx1CurrentCSSs'         => 0,
#    'dsx1CurrentPCVs'         => 0,
#    'dsx1CurrentLESs'         => 0,
#    'dsx1CurrentBESs'         => 0,
#    'dsx1CurrentDMs'          => 0,
#    'dsx1CurrentLCVs'         => 0,
#    'isdnEndpointStatus'      => 'inactive',
#    'isdnSignalingProtocol'   => 'dms100',
#    'isdnSignalingStatus'     => 'inactive',
#  );

my %errors;                    # Hash of hash refs, keyed by target
#  e.g.
#  %errors = (
#    target1 => {
#      1 => 0,
#      2 => 0,
#      3 => 0,
#    },
#    target2 => {
#      15 => 0,
#      21 => 0,
#      33 => 0,
#    }


# Define global variables
$program_name = 'wan-circuit-alarm';
$usage = 'Usage:  wan-circuit-alarm -s {yes|no} [-d {integer}] [-r] [-a | -e {expr} | -f {filename} | target1 target2 target3 ...]'; 
$version = '1.2.0';

# Define the IANA ifTypes which designate a WAN circuit
@wan_if_types = qw/ds1 e1 basicISDN primaryISDN propPointToPointSerial ds3/;


# Grab arguments
getopts('ad:e:f:rs:', \%option); 
@target = @ARGV;

# Set mode
if      ($option{r})            { $mode = 'report'      }
elsif   (-t STDIN)              { $mode = 'interactive' }
else                            { $mode = 'batch'       }


### Begin Main Program ###############################################

{
  check_args();                   # Check arguments

  read_config();                  # Read Netops config file

  compile_mibs();                 # Compile MIB files

  build_target();                 # Populate @target

  target_check();                 # Look for errors in @target

  basic_info();                   # Gather information

  info_before();                  # Gather more information

  sanity_check();                 # Check for error conditions

  do_the_work();                  # Do it

  identify_alarms();              # Count devices with alarms

  write_alarm_log();              # Record issues

  print_report();                 # Print report

  notify_staff();                 # Mail report
}

##### End Main Program #################################################



########################################################################
# Query variables
########################################################################
sub do_the_work {
  my %arg;
  my $num;

  # Debug trace
  trace_location('begin') if $debug;

  # Notify operator
  print_it('Looking for alarms...');
  unless ($dome) {
    sleep $short;
    return 1;
  }

  # Pause a bit to give ifErrors time to increment
  say "  Pausing $long seconds before gathering error counters again" if $debug;
  sleep $long;

  # Loop through the list of targets
  for my $target (@target) {
    say "Processing $target" if $debug;

    # Walk interfaces
    for my $ifIndex (keys %{$if_info{$target}}) {
      my ($dsxInstance, $ifInstance, $ifName);
      $ifName = $if_info{$target}->{$ifIndex}->{'ifName'};

      # Extract IF-MIB and DS1-MIB interface status
      $ifInstance  = $if_info{$target}->{$ifIndex};
      $dsxInstance = $if_info{$target}->{$ifIndex};

      # Debug info
      say "  Examining $ifName at ifIndex.$ifIndex" if $debug;

      # Calculate errors.  I don't understand the dsx1 errors, so ignore them
      $errors{$target}->{$ifIndex} += $ifInstance->{'ifInErrors'};
      $errors{$target}->{$ifIndex} += $ifInstance->{'ifOutErrors'};
#      $errors{$target}->{$ifIndex} += $dsxInstance->{'dsx1CurrentSESs'};
#      $errors{$target}->{$ifIndex} += $dsxInstance->{'dsx1CurrentSEFSs'};
#      $errors{$target}->{$ifIndex} += $dsxInstance->{'dsx1CurrentUASs'};
#      $errors{$target}->{$ifIndex} += $dsxInstance->{'dsx1CurrentCSSs'};
#      $errors{$target}->{$ifIndex} += $dsxInstance->{'dsx1CurrentPCVs'};
#      $errors{$target}->{$ifIndex} += $dsxInstance->{'dsx1CurrentLESs'};
#      $errors{$target}->{$ifIndex} += $dsxInstance->{'dsx1CurrentLESs'};
#      $errors{$target}->{$ifIndex} += $dsxInstance->{'dsx1CurrentDMs'};
#      $errors{$target}->{$ifIndex} += $dsxInstance->{'dsx1CurrentLCVs'};

      # Skip interfaces which are administratively disabled
      if ($ifInstance->{'ifAdminStatus'} eq 'down') {
        say "    $ifName is administratively down, ignoring alarms" if $debug;
      }

      # Skip interfaces for which the description includes the string 'unused'
      elsif ($ifInstance->{'ifAlias'} =~ /unused/i) {
        say "    $ifName is described as 'unused', ignoring alarms" if $debug;
      }

      # Count alarms
      else {
        my $val;

        # Consider ifOperStatus
        unless ($ifInstance->{'ifOperStatus'} eq 'up') {
          $alarm_count{$target}++;
          $val = $ifInstance->{'ifOperStatus'};
          log_it("ifOperStatus = $val on $target");
          say "    In alarm because ifOperStatus = $val" if $debug;
        }

        # Consider dsx1LineStatus
        unless ($dsxInstance->{'dsx1LineStatus'} eq 'NoAlarm') {
          unless ($dsxInstance->{'dsx1LineStatus'} eq $DASH) {
            $alarm_count{$target}++;
            $val = $ifInstance->{'dsx1LineStatus'};
            log_it("dsx1LineStatus = $val on $target");
            say "    In alarm because dsx1LineStatus = $val" if $debug;
          }
        }

        # Consider isdnSignalingStatus
        unless ($dsxInstance->{'isdnSignalingStatus'} eq 'active') {
          unless ($dsxInstance->{'isdnSignalingStatus'} eq $DASH) {
            $alarm_count{$target}++;
            $val = $ifInstance->{'isdnSignalingStatus'};
            log_it("isdnSignalingStatus = $val on $target");
            say "    In alarm because isdnSignalingStatus = $val" if $debug;
          }
        }

        # Consider isdnEndpointStatus
        unless ($dsxInstance->{'isdnEndpointStatus'} eq 'active') {
          unless ($dsxInstance->{'isdnEndpointStatus'} eq $DASH) {
            $alarm_count{$target}++;
            $val = $ifInstance->{'isdnEndpointStatus'};
            log_it("isdnEndPointStatus = $val on $target");
            say "    In alarm because isdnEndpointStatus = $val" if $debug;
          }
        }

        # Consider IF errors
        if ($errors{$target}->{$ifIndex} > 0) {
          my ($latest, $previous);
          $previous = $errors{$target}->{$ifIndex};

          # Gather error counters from this interface again
          $latest = sum_if_errors($target, $ifIndex);    
        
          # If error counters haven't changed, ignore
          if ($latest == $previous) {
            # Do nothing
          }

          # Otherwise, increment alarm counter
          else {
            $alarm_count{$target}++;
            log_it("Error counters on $target: $ifName are incrementing:  from $previous to $latest");
            say "    In alarm because errors = $latest" if $debug;
            $errors{$target}->{$ifIndex} = $latest;
          }

        }  # End 'Consider IF errors'

      }  # End 'Count alarms'

    }  # End 'Walk interfaces'

    # Entertain operator
    print $BANG if $mode eq 'interactive';

  }  # End 'Loop through targets'

  # Make things look pretty
  say "\n" if $mode eq 'interactive';

  # Debug trace
  trace_location('end') if $debug;

  return 1;  
}




########################################################################
# Gather information ... and perform more error checking
########################################################################
sub info_before {

  # Debug trace
  trace_location('begin') if $debug;

  # Notify operator
  print_it('Gathering more information...');

  # Loop through targets, gathering more information
  TARGET:
  for my $target (@target) {
    my (@channels, $vb);
    say "Processing $target" if $debug;

    # Find WAN interfaces by walking ifType
    $vb = snmpWalk( {host => $target, oid =>  'ifType'} );
    VARBIND:
    for my $varbind (@$vb) {
      my ($if_type, $iid);
      $iid = $varbind->{iid};
      $if_type = $varbind->{val};

      # If this looks like a WAN interface
      if ( any { $_ eq $if_type } @wan_if_types ) {
        say "$if_type looks like a WAN interface" if $debug > 1;

        # Skip it if it looks like a channel
        my $if_descr = snmpGet( {host => $target, oid => "$ifDescr_oid.$iid"} );
        unless ($if_descr =~ /Signaling/) {
          if ($if_descr =~ /:/) {
            say '  This interface belongs to a channel, skip it' if $debug > 1;
            next VARBIND;
          }
        }

        # Save what I found
        $if_info{$target}->{$iid}->{'ifType'} = $if_type;
        $if_info{$target}->{$iid}->{'ifDescr'} = $if_descr;

        # Debug info
        say "  Found WAN interface '$if_type' at ifIndex $iid" if $debug > 1;
      }
    }

    # Cycle through ifIndex, adding IF-MIB parameters
    for my $ifIndex (keys %{$if_info{$target}}) {
      say "Gathering IF-MIB values for ifIndex $ifIndex" if $debug > 1;

      # Add ifAlias
      $if_info{$target}->{$ifIndex}->{'ifAlias'} 
          = snmpGet( { host=> $target, oid => "ifAlias.$ifIndex"} );

      # Add ifName
      say "Getting ifName.$ifIndex" if $debug > 3;
      $if_info{$target}->{$ifIndex}->{'ifName'} 
          = snmpGet( {host => $target, oid => "$ifName_oid.$ifIndex"} );

      # Add ifAdminStatus
      $if_info{$target}->{$ifIndex}->{'ifAdminStatus'}
          = snmpGet( {host => $target, oid => "ifAdminStatus.$ifIndex"} );

      # Add ifOperStatus
      $if_info{$target}->{$ifIndex}->{'ifOperStatus'}
          = snmpGet( {host => $target, oid => "ifOperStatus.$ifIndex"} );

      # Add ifInErrors
      $if_info{$target}->{$ifIndex}->{'ifInErrors'}
          = snmpGet( {host => $target, oid => "ifInErrors.$ifIndex"} );
      $if_info{$target}->{$ifIndex}->{'ifInErrors'} //= 0;

      # Add ifOutErrors
      $if_info{$target}->{$ifIndex}->{'ifOutErrors'}
          = snmpGet( {host => $target, oid => "ifOutErrors.$ifIndex"} );
      $if_info{$target}->{$ifIndex}->{'ifOutErrors'} //= 0;

     
    }  # End 'supports IF-MIB'


    # Cycle through ifIndex, adding DS1-MIB parameters
    for my $ifIndex (keys %{$if_info{$target}}) {
      my ($ifName, $val);
      $ifName = $if_info{$target}->{$ifIndex}->{'ifName'};
      say "  Processing $ifName at iid $ifIndex for DS1-MIB vars" if $debug > 2;

      # Check for DS1-MIB support by getting dsx1LineIndex.$ifIndex
      $val = snmpGet( {host => $target, oid => "dsx1LineIndex.$ifIndex"} );

      # If this interface supports DS1-MIB, gather parameters
      if (defined $val and $val ne $EMPTY_STR) {
        say "  Gathering DS1-MIB values for $ifName / $ifIndex" if $debug > 1;

        # Add dsx1LineType
        $if_info{$target}->{$ifIndex}->{'dsx1LineType'}
          = snmpGet( {host => $target, oid => "dsx1LineType.$ifIndex"} );
  
        # Add dsx1LineStatus
        $if_info{$target}->{$ifIndex}->{'dsx1LineStatus'}
            = get_dsx1_line_status($target, $ifIndex);
  
#        I don't understand dsx1 errors, so ignore them --sk
#        # Add dsx1CurrentESs
#        $if_info{$target}->{$ifIndex}->{'dsx1CurrentESs'}
#          = get_dsx1_current_ESs($target, $ifIndex);
#  
#        # Add dsx1CurrentSESs
#        $if_info{$target}->{$ifIndex}->{'dsx1CurrentSESs'}
#            = get_dsx1_current_SESs($target, $ifIndex);
#  
#        # Add dsx1CurrentSEFSs
#        $if_info{$target}->{$ifIndex}->{'dsx1CurrentSEFSs'}
#            = get_dsx1_current_SEFSs($target, $ifIndex);
#  
#        # Add dsx1CurrentUASs
#        $if_info{$target}->{$ifIndex}->{'dsx1CurrentUASs'}
#            = get_dsx1_current_UASs($target, $ifIndex);
#  
#        # Add dsx1CurrentCSSs
#        $if_info{$target}->{$ifIndex}->{'dsx1CurrentCSSs'}
#            = get_dsx1_current_CSSs($target, $ifIndex);
#  
#        # Add dsx1CurrentPCVs
#        $if_info{$target}->{$ifIndex}->{'dsx1CurrentPCVs'}
#            = get_dsx1_current_PCVs($target, $ifIndex);
#  
#        # Add dsx1CurrentLESs
#        $if_info{$target}->{$ifIndex}->{'dsx1CurrentLESs'}
#            = get_dsx1_current_LESs($target, $ifIndex);
#  
#        # Add dsx1CurrentBESs
#        $if_info{$target}->{$ifIndex}->{'dsx1CurrentBESs'}
#            = get_dsx1_current_BESs($target, $ifIndex);
#
#        # Add dsx1CurrentDMs
#        $if_info{$target}->{$ifIndex}->{'dsx1CurrentDMs'}
#            = get_dsx1_current_DMs($target, $ifIndex);
#
#        # Add dsx1CurrentLCVs
#        $if_info{$target}->{$ifIndex}->{'dsx1CurrentLCVs'}
#            = get_dsx1_current_LCVs($target, $ifIndex);

      }  # End 'supports DS1-MIB'

      # Fill with dashes and zeros
      else {
        say '  Zeroing DS1-MIB values' if $debug > 1;
        $if_info{$target}->{$ifIndex}->{'dsx1LineType'}     = $DASH;
        $if_info{$target}->{$ifIndex}->{'dsx1LineStatus'}   = $DASH;
        $if_info{$target}->{$ifIndex}->{'dsx1CurrentESs'}   = 0;
        $if_info{$target}->{$ifIndex}->{'dsx1CurrentSESs'}  = 0;
        $if_info{$target}->{$ifIndex}->{'dsx1CurrentSEFSs'} = 0;
        $if_info{$target}->{$ifIndex}->{'dsx1CurrentUASs'}  = 0;
        $if_info{$target}->{$ifIndex}->{'dsx1CurrentCSSs'}  = 0;
        $if_info{$target}->{$ifIndex}->{'dsx1CurrentPCVs'}  = 0;
        $if_info{$target}->{$ifIndex}->{'dsx1CurrentLESs'}  = 0;
        $if_info{$target}->{$ifIndex}->{'dsx1CurrentDMs'}   = 0;
        $if_info{$target}->{$ifIndex}->{'dsx1CurrentLCVs'}  = 0;
      }  # End 'Fill with dashes and zeros'

    }  # End DS1-MIB


    # Cycle through ifIndex, adding ISDN-MIB parameters
    say 'Testing for ISDN-MIB support' if $debug;
    my @ifIndex = keys %{$if_info{$target}};      # Work around SNMP.pm bug
    for my $ifIndex (@ifIndex) {
      my ($ifName, $vb);
      $ifName = $if_info{$target}->{$ifIndex}->{'ifName'};
      say "  Processing $ifName at iid $ifIndex for ISDN-MIB vars" if $debug>2;

      # Check for ISDN-MIB support by walking isdnSignalingIfIndex.$ifIndex
      $vb = snmpWalk( {host => $target, oid => 'isdnSignalingIfIndex'} );

      # If this target supports ISDN-MIB, gather parameters
      if (@$vb > 0) {
        my $isdn_index;
        say "  Gathering ISDN-MIB values for $ifName: $ifIndex" if $debug > 1;

        # Find isdnSignalingIfIndex
        $isdn_index = find_isdn_signaling_if_index($target, $ifIndex);
        $isdn_index = $DASH unless defined $isdn_index;
        $if_info{$target}->{$ifIndex}->{'isdnSignalingIfIndex'} = $isdn_index;

        # Add ISDN parameters
        if ($isdn_index eq $QUERY) {
          $if_info{$target}->{$ifIndex}->{'isdnSignalingProtocol'} = $DASH;
          $if_info{$target}->{$ifIndex}->{'isdnSignalingStatus'}   = $DASH;
          $if_info{$target}->{$ifIndex}->{'isdnEndpointStatus'}    = $DASH;
        }
        else {
          $if_info{$target}->{$ifIndex}->{'isdnSignalingStatus'}
            = snmpGet({host=>$target, oid=>"isdnSignalingStatus.$isdn_index"});
          $if_info{$target}->{$ifIndex}->{'isdnEndpointStatus'}
            = snmpGet({host=>$target, oid=>"isdnEndpointStatus.$isdn_index"});
          $if_info{$target}->{$ifIndex}->{'isdnSignalingProtocol'}
            = snmpGet({host=>$target, oid=>"isdnSignalingProtocol.$isdn_index"});
        }

      }  # End 'supports ISDN-MIB'

      # Fill with dashes
      else {
        say '  Zeroing ISDN-MIB values' if $debug > 1;
        $if_info{$target}->{$ifIndex}->{'isdnSignalingProtocol'} = $DASH;
        $if_info{$target}->{$ifIndex}->{'isdnSignalingStatus'}   = $DASH;
        $if_info{$target}->{$ifIndex}->{'isdnEndpointStatus'}    = $DASH;
      }

    }  # End ISDN-MIB

    # Entertain operator
    print $BANG if $mode eq 'interactive';
  }

  # Make things look pretty
  say "\n" if $mode eq 'interactive';

  # Debug trace
  trace_location('end') if $debug;

  return 1;
}




########################################################################
# Tell the operator what I discovered
########################################################################
sub print_report {
  my $handle;
  my $total = @target;
  my $now = get_now();

  # If we are running in test mode, skip this routine
  unless ($dome) {
    print_it("Running in test mode, cannot print a meaningful report\n");
    return 1;
  }

  # Debug trace
  trace_location('begin') if $debug;

  # Direct output to screen or to file
  if ($mode eq 'interactive') {
    $handle = *STDOUT;
  }
  else {
    open $handle, '>', $report_file or die "Cannot open $report_file: $!\n";
  }

print {$handle} <<EOF;
# Title:           $report_subject
#
# Institution:     $institution
#
# Date of Report:  $now
#
# Description:     This report lists the status of WAN circuits
#
#                  Look through syslog for the details of alarms:
#                       grep $program_name /var/log/syslog
#
# Active:          $total
#
# Alarm Count:     $shit_happens
#
# In Alarm:        @alarm_list
# 
# Questions:       If you have questions or comments regarding this
#                  report, please mail them to $report_queries.
#
# Name             = ifAlias or ifName
# Protocol         = isdnSignalingProtocol
# Type             = ifType (prop = propPointToPointSerial)
# dsx1LineStatus   = dsx1LineStatus (fine, LossOfFrame, LoopBack, etc.)
# ifAd             = ifAdminStatus (up, down, test)
# ifOp             = ifOperStatus (up, down, test)
# St               = isdnSignalingStatus (up, dn)
# En               = isdnEndpointStatus (up, dn)
# Errors           = sum of numerous DS1-MIB error counters
#
target          Name         Type Protocol dsx1LineStatus ifAd ifOp St En Errors
--------------- ------------ ---- -------- -------------- ---- ---- -- -- ------
EOF

  # Walk targets
  for my $target (@target) {
    printf {$handle} 
      "%-15s ", $target;
    my $first = 1;

    # Walk interfaces
    INTERFACE:
    for my $index (keys%{$if_info{$target}}) {
      my ($ifStuff, $alias, $ifAdminStatus, $ifOperStatus);
      my ($ifType, $protocol, $dsx1LineStatus);
      my ($isdn_status, $isdn_end_status, $errors);
      $ifStuff         = $if_info{$target}->{$index};
      $alias           = $ifStuff->{'ifAlias'};
      $alias           = $ifStuff->{'ifName'} if $alias eq $EMPTY_STR;
      $ifType          = substr($ifStuff->{'ifType'}, 0, 4);
      $protocol        = substr($ifStuff->{'isdnSignalingProtocol'}, 0, 8);
      $dsx1LineStatus  = substr($ifStuff->{'dsx1LineStatus'}, 0, 14);
      $ifAdminStatus   = $ifStuff->{'ifAdminStatus'};
      $ifOperStatus    = $ifStuff->{'ifOperStatus'};
      $isdn_status     = $ifStuff->{'isdnSignalingStatus'};
      $isdn_end_status = $ifStuff->{'isdnEndpointStatus'};
      $errors          = $errors{$target}->{$index};
      next INTERFACE if $ifAdminStatus eq 'down';

      # Mangle into smaller strings
      $alias = substr($alias, 0, 12);
      given ($dsx1LineStatus) {
        when ('NoAlarm')   { $dsx1LineStatus = 'fine' }
      }
      given($isdn_status) {
        when ('active')   { $isdn_status = 'up' }
        when ('inactive') { $isdn_status = 'dn' }
      }
      given($isdn_end_status) {
        when ('active')   { $isdn_end_status = 'up' }
        when ('inactive') { $isdn_end_status = 'dn' }
      }

      # Print first line for this target
      if ($first == 1) {
        printf {$handle} "%-12s %-4s %-8s %-14s %-4s %-4s %-2s %-2s %-6s\n", 
            $alias, $ifType, $protocol, $dsx1LineStatus, 
            $ifAdminStatus, $ifOperStatus, $isdn_status, $isdn_end_status,
            $errors;
        $first = 0;
      }

      # Print subsequent lines for this target
      else {
        printf {$handle} "                %-12s %-4s %-8s %-14s %-4s %-4s %-2s %-2s %-6s\n", 
            $alias, $ifType, $protocol, $dsx1LineStatus, 
            $ifAdminStatus, $ifOperStatus, $isdn_status, $isdn_end_status,
            $errors;
      }

    }  # End 'Walk interfaces'

  }  # End 'Walk targets'

  # Add silent devices to the report
  for my $silent (sort @silent) {
    printf {$handle} "%-15s  Not answering pings\n", $silent;
  }

  # Add unresponsive devices to the report
  for my $unresponsive (sort @unresponsive) {
    printf {$handle} "%-15s  Not answering SNMP GETs\n", $unresponsive;
  }

  # Add insane devices to the report
  for my $insane (sort @insane) {
    printf {$handle} "%-15s  Insane\n", $insane;
  }

  # Clean up
  unless ($handle =~ /STDOUT/) {
    close $handle or warn "Cannot close $report_file: $!\n";
  }

  # Debug trace
  trace_location('end') if $debug;

  return 1;
}




########################################################################
# Remove entries.  Sometimes, the operator gives us target(s) which I
# cannot process. Instead of crashing & burning, remove these
# from the list
########################################################################
sub prune_local {
  my @nuked;
  my @remove = @_;

  # Debug trace
  trace_location('begin') if $debug;

  # Check to see if we have work to do
  if (@remove > 0) {

    # Make things look pretty
    say('') if $mode eq 'interactive';

    # Remove entries which failed checks
    for my $remove (@remove) {
      say "Removing $remove" if $debug;
      delete $if_info{$remove};
      delete $errors{$remove};
      push @nuked, $remove;
   }
  }

  # Debug trace
  trace_location('end') if $debug;

  return @nuked;
}




########################################################################
# Sanity check
########################################################################
sub sanity_check {
  my @remove;

  # Debug trace
  trace_location('begin') if $debug;

  # Notify operator
  print_it('Sanity check...');

  # Loop through targets, looking for problems
  TARGET:
  for my $target (@target) {

    # If target contains no WAN interfaces, remove it
    if (keys %{$if_info{$target}} == 0) {
      say "$target reports no serial interfaces, ignoring" if $debug;
      push @remove, $target;
      next TARGET;
    }

    # Entertain operator
    print $BANG if $mode eq 'interactive';
  }

  # Remove entries which failed checks
  prune_basic(@remove);

  # Make things look pretty
  say "\n" if $mode eq 'interactive';
  
  # Debug trace
  trace_location('end') if $debug;
    
  return 1;
}     
    

    
    
########################################################################
# Output help
########################################################################
sub HELP_MESSAGE {
  print <<EOF;

  See $netops_global_config (and $netops_alarm_config if this is an
  xxxx-alarm script) for configuration options

  $usage

    -s asks the question:  are you serious?  If you answer 'no', then 
       the script will run in demo mode, making no changes

    -d specifies debug level (typically 0-9)

    -r determines whether or not the script writes its report to disk

    -a instructs the script to filter the hosts table using its internally
       defined suffixes variable to produce a list of targets

    -e requires a regex as its argument, with which the script filters
       the hosts table to produce a list of targets.  Enclose the regex in 
       quotes to protect special characters from the shell

    -f points the script to a file containing the list of targets

    target1 target2 target3 ... is the command-line list of targets you
      have specified, in lieu of the -a, -e, or -f options

EOF

  return 1;
}




########################################################################
# Output version
########################################################################
sub VERSION_MESSAGE {
  say "$program_name v$version";
  return 1;
}